Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize one bounded synchronization interval, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false, None).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true, None).await
1357    }
1358
1359    pub(crate) async fn authorize_base_recovery_control(
1360        home: &Path,
1361        task: &str,
1362        control: &mut crate::base_recovery_control::Transaction,
1363    ) -> Result<()> {
1364        let id = control.request.run.clone();
1365        Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366    }
1367
1368    async fn authorize_base_recovery_request(
1369        home: &Path,
1370        id: &str,
1371        task: &str,
1372        resumption: bool,
1373        mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374    ) -> Result<()> {
1375        let _owner = crate::pr_head::owner_lock(home, id)?;
1376        let mut state = RunState::load_under(id, home)?;
1377        crate::pr_head::ensure_owner(&state)?;
1378        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379        anyhow::ensure!(
1380            queued.execution_runs().last().is_some_and(|run| run == id)
1381                && queued.status != crate::queue::TaskStatus::Done
1382                && !queued.fresh_start
1383                && queued.review_branch.is_none(),
1384            "run task ownership changed"
1385        );
1386        anyhow::ensure!(
1387            state
1388                .origin
1389                .as_ref()
1390                .and_then(|o| o.task.as_deref())
1391                .is_none_or(|owner| owner == task),
1392            "run belongs to another task"
1393        );
1394        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395        anyhow::ensure!(
1396            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397            "run is live or its ownership is unknown"
1398        );
1399        // Judge the previous driver's lease before acquiring our maintenance
1400        // lease. The owner lock excludes new driver claims across both steps.
1401        let _execution = crate::driver::maintenance(home, id)?;
1402        anyhow::ensure!(
1403            state.status.resumable()
1404                && state.released_to.is_none()
1405                && state.retired_to.is_none()
1406                && state.pr_taken_over_by.is_none(),
1407            "run ownership is final"
1408        );
1409        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410        anyhow::ensure!(
1411            !state.pr_publication_pending
1412                && !crate::handover::landing_checkpoint(&state)
1413                && state.merge_decision.is_none(),
1414            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415        );
1416        let winner = state.winner().context("run has no retained winner")?;
1417        anyhow::ensure!(
1418            git::is_clean(&winner.worktree).await?,
1419            "retained worktree has uncommitted changes"
1420        );
1421        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422        anyhow::ensure!(
1423            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424            "retained worktree does not match the winner branch"
1425        );
1426        if let Some(control) = control.as_ref() {
1427            anyhow::ensure!(
1428                head.starts_with(&control.request.head),
1429                "retained HEAD changed since the owner request"
1430            );
1431            anyhow::ensure!(
1432                state
1433                    .base_sync_recovery
1434                    .stop
1435                    .as_ref()
1436                    .is_none_or(|stop| stop.head == head),
1437                "retained HEAD differs from the recorded stop head; inspect it first"
1438            );
1439        }
1440        state.ensure_base_episode();
1441        state.record_base_sync_stop(&head);
1442        anyhow::ensure!(
1443            state
1444                .base_sync_recovery
1445                .stop
1446                .as_ref()
1447                .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448            "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449        );
1450        let fetched =
1451            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452        anyhow::ensure!(
1453            fetched.ok(),
1454            "cannot authorize base recovery without a fetched base"
1455        );
1456        let approved_tip = git::rev_parse(
1457            &state.repo,
1458            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459        )
1460        .await?;
1461        if let Some(control) = control.as_deref_mut() {
1462            let queue = crate::queue::Queue::at(home.join("queue"));
1463            control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464            if let Some(prior) = state
1465                .base_sync_recovery
1466                .authorizations
1467                .iter()
1468                .find(|a| a.request.as_deref() == Some(&control.key()))
1469            {
1470                anyhow::ensure!(
1471                    !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472                    "recovery request was consumed or invalidated; it cannot queue again"
1473                );
1474                return control.enqueue(&queue);
1475            }
1476        }
1477        if resumption {
1478            anyhow::ensure!(
1479                crate::daemon::compatible_run_instruction(&state, &queued),
1480                "resumption instruction does not match the retained run"
1481            );
1482            let approval = queued
1483                .resumption
1484                .as_ref()
1485                .context("no operator resumption")?;
1486            anyhow::ensure!(
1487                approval.instruction == queued.instruction,
1488                "resumption approval does not match the current task instruction"
1489            );
1490            state.authorize_base_sync_resumption(task, approval)?;
1491        } else {
1492            state.authorize_base_sync(task)?;
1493        }
1494        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495        state
1496            .base_sync_recovery
1497            .authorizations
1498            .last_mut()
1499            .unwrap()
1500            .scope = Some(crate::base_sync_recovery::Scope {
1501            tip: approved_tip,
1502            instruction: queued.instruction.clone(),
1503            episode,
1504        });
1505        if let Some(control) = control.as_deref_mut() {
1506            let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507            authorization.request = Some(control.key());
1508            authorization.generation = control.generation();
1509            authorization.decision = control.decision();
1510            authorization.question = control
1511                .request
1512                .decision
1513                .as_ref()
1514                .map(|_| control.request.request.clone());
1515        }
1516        state.event(
1517            "base-recovery",
1518            "owner authorized a bounded base synchronization interval; fresh verification required",
1519        );
1520        state.save_under(home)?;
1521        if let Some(control) = control {
1522            control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523        }
1524        Ok(())
1525    }
1526
1527    /// Reopen an existing run.
1528    pub fn resume(id: &str) -> Result<Self> {
1529        let state = RunState::load(id)?;
1530        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531        if state.startup.as_ref().is_some_and(|s| !s.complete)
1532            || (state.status == RunStatus::Prep
1533                && state.instruction.trim().is_empty()
1534                && state.candidates.is_empty()
1535                && state.prep_complete == Some(false))
1536        {
1537            bail!(
1538                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539            );
1540        }
1541        crate::pr_head::ensure_owner(&state)?;
1542        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543            bail!(
1544                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545            );
1546        }
1547        if let Some(to) = &state.released_to {
1548            bail!(
1549                "run {} cannot be resumed: its worktree was released to run {}",
1550                state.short(),
1551                crate::run::short_of(to)
1552            );
1553        }
1554        let roles = state.config.resolve_roles()?;
1555        if let Some(id) = &state.config.graph.selected_implementer
1556            && state.candidates.iter().any(|c| c.agent != *id)
1557        {
1558            bail!(
1559                "replacement implementer pin does not match the saved candidates; task must stay held"
1560            );
1561        }
1562        let max_parallel = state.config.graph.max_parallel.max(1);
1563        Ok(Self {
1564            state,
1565            roles,
1566            sem: Arc::new(Semaphore::new(max_parallel)),
1567            pause: Pause::new(),
1568            interrupt: Pause::new(),
1569        })
1570    }
1571
1572    /// Walk the graph to a terminal state, skipping nodes already recorded.
1573    ///
1574    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1575    /// the phone - ends here, so this is the one place a run that ended
1576    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1577    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1578    pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579        // Allocate before polling: boxing inside an async wrapper would still
1580        // retain its construction frame while the nested graph is polled.
1581        Box::pin(self.execute_inner())
1582    }
1583
1584    async fn execute_inner(&mut self) -> Result<()> {
1585        anyhow::ensure!(
1586            self.state.startup.as_ref().is_none_or(|s| s.complete),
1587            "startup incomplete; inspect retained evidence before explicit recovery"
1588        );
1589        if let Some(selection) = &self.state.pr_recovery
1590            && (selection.source_proof.is_some() || selection.source_run.is_some())
1591        {
1592            Box::pin(selection.verify_source(&self.state.repo)).await?;
1593        }
1594        // Queue claims cover daemon work; this also serializes manual drivers
1595        // consuming a repair answer, including the new review and decision.
1596        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597            || self.state.land_approval.is_some()
1598            || self.state.merge_decision.is_some()
1599            || !self.state.merge_repairs.is_empty()
1600            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601        {
1602            let claim = FixClaim::acquire(&self.state.dir())?;
1603            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604            let disk = RunState::load(&self.state.id)?;
1605            let checkpoint = |s: &RunState| {
1606                (
1607                    s.merge_repairs.len(),
1608                    s.merge_repairs
1609                        .last()
1610                        .map(|r| (r.question.clone(), r.phase)),
1611                    s.land_approval
1612                        .as_ref()
1613                        .map(|a| (a.question.clone(), a.head.clone())),
1614                )
1615            };
1616            if checkpoint(&disk) != checkpoint(&self.state) {
1617                bail!(
1618                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619                );
1620            }
1621            Some(claim)
1622        } else {
1623            None
1624        };
1625        let mut execution = self.claim_driver().await?;
1626        let runner = &mut *execution.runner;
1627        let wait = runner.repository_wait();
1628        // Keep the graph's large future off the callers' stack, including
1629        // standalone task attempts running on a default-sized test thread.
1630        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631        let result = match result {
1632            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633                record_repository_block(&mut runner.state, &e);
1634                runner.park_here()?;
1635                Ok(())
1636            }
1637            other => other,
1638        };
1639        execution.driver.finish(&mut runner.state).await?;
1640        let ended = if result.is_err() {
1641            Some({
1642                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643                if let Err(error) = &result {
1644                    notice.message = format!(
1645                        "Run {} stopped with an error: {}",
1646                        runner.state.id,
1647                        error.to_string().lines().next().unwrap_or("unknown error")
1648                    );
1649                }
1650                notice
1651            })
1652        } else {
1653            crate::notices::run_ended(&runner.state)
1654        };
1655        if let Some(notice) = ended {
1656            crate::notices::raise(notice);
1657        }
1658        result
1659    }
1660
1661    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663    }
1664
1665    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666        let wait = self.repository_wait();
1667        coordinate_operation(&mut self.state, cwd, &wait).await
1668    }
1669
1670    /// Validate ownership and persist the driver claim in one critical section.
1671    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672        let owner = match crate::pr_head::owner_lock_wait(
1673            &crate::run::home(),
1674            &self.state.id,
1675            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676        )
1677        .await
1678        {
1679            Ok(owner) => owner,
1680            Err(error) => {
1681                let detail = format!("worktree maintenance or ownership update: {error:#}");
1682                // Admission did not own run.json: retain this resource outcome
1683                // in memory for queue settlement without overwriting a concurrent
1684                // retirement or publication transaction.
1685                self.state.resource_block = Some(crate::run::ResourceBlock {
1686                    cause: crate::run::ResourceBlockCause::Other,
1687                    node: "start".into(),
1688                    kind: "worktree maintenance".into(),
1689                    detail: detail.clone(),
1690                    at: Timestamp::now(),
1691                });
1692                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693            }
1694        };
1695        let disk = RunState::load(&self.state.id)?;
1696        crate::pr_head::ensure_owner(&disk)?;
1697        let claimed =
1698            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699        // Only proven-live ownership refuses here. Unknown records (legacy runs
1700        // with no claim marker or process identity) fall through to the
1701        // exclusive execution lease taken just below, which is what actually
1702        // keeps a second driver out.
1703        anyhow::ensure!(
1704            disk.liveness(claimed) != crate::run::Liveness::Live,
1705            "run already has a live driver; wait for its execution to end"
1706        );
1707        if let Some(to) = &disk.released_to {
1708            bail!(
1709                "run {} cannot continue: its worktree was released to run {}",
1710                self.state.short(),
1711                crate::run::short_of(to)
1712            );
1713        }
1714        anyhow::ensure!(
1715            serde_json::to_value(&disk.base_sync_recovery)?
1716                == serde_json::to_value(&self.state.base_sync_recovery)?,
1717            "base recovery changed since this run was loaded; reload the saved run"
1718        );
1719        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720        let mut execution = DriverExecution {
1721            runner: self,
1722            driver,
1723        };
1724        // Declare after the execution guard: on any claim error this short
1725        // lock must drop before that guard reloads and releases ownership.
1726        let _owner = owner;
1727        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728            .await?;
1729        let state = &mut execution.runner.state;
1730        // Moving again, so it is no longer parked. Set before the walk rather
1731        // than in `resume`, so every way of re-entering the graph clears it
1732        // and a card cannot claim a run is waiting to be resumed while the
1733        // agents are already working.
1734        state.parked = false;
1735        state.resource_wait = None;
1736        state.resource_block = None;
1737        // Any seat this state still lists as answering belongs to whatever
1738        // process last drove this run — this one included, if it crashed
1739        // mid-wave. Cleared and flushed immediately, before anything else
1740        // runs, so a resume can never show a seat as live when nothing is
1741        // asking it anything yet; the node that actually dispatches the next
1742        // wave repopulates it.
1743        state.clear_active();
1744        // Recorded in the same spot, and flushed together with the clear
1745        // above: this is the pid a reader checks (`RunState::liveness`) when
1746        // no daemon claim exists to answer "is a process still driving this
1747        // run" — a plain `magi run` / `magi review` typed into a terminal
1748        // claims nothing there. Always overwritten, never only-if-absent, so
1749        // a resumed run's stale pid from a previous, possibly-dead process
1750        // can never survive into this one's own report. Unlike
1751        // `clear_active`, this changes on every single `execute()` call, so
1752        // the save below is now unconditional rather than only-if-cleared.
1753        //
1754        // `driver_started_at` is recorded in the same breath, from this same
1755        // pid, so `liveness` can tell a live pid that is genuinely still us
1756        // apart from one the OS has since handed to an unrelated process —
1757        // see that field's own doc for why the pid alone is not enough.
1758        let pid = std::process::id();
1759        state.driver_pid = Some(pid);
1760        state.driver_started_at = crate::proc::process_started_at(pid);
1761        state.driver_exited = false;
1762        state.driver_claimed = Some(true);
1763        state.begin_pass();
1764        execution.driver.bind(state);
1765        state.save()?;
1766        execution.driver.record(state)?;
1767        Ok(execution)
1768    }
1769
1770    /// No conflict checkpoint proves dispatch was never reached. End that
1771    /// consumed generation so an owner can explicitly authorize a fresh one.
1772    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774        if let Err(error) = &result
1775            && !error.is::<crate::repo_lock::ResourceBlocked>()
1776            && self
1777                .state
1778                .repair_grants
1779                .last()
1780                .is_some_and(|g| g.conflict.is_none())
1781        {
1782            self.state.repair_grants.last_mut().unwrap().phase =
1783                crate::repair_budget::Phase::Complete;
1784            self.state.status = RunStatus::Blocked;
1785            self.state.save()?;
1786            bail!(
1787                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788                self.state.passes.last().map_or(0, |p| p.index)
1789            );
1790        }
1791        result
1792    }
1793
1794    /// A grant owns only these repair attempts. Confirmation reviews cannot
1795    /// create another fixer allowance, and every operation is restart-checkpointed.
1796    async fn execute_granted_repair(&mut self) -> Result<()> {
1797        use crate::repair_budget::{Kind, Phase};
1798        let index = self.state.repair_grants.len() - 1;
1799        let winner = self
1800            .state
1801            .winner()
1802            .cloned()
1803            .context("repair winner missing")?;
1804        loop {
1805            let grant = self.state.repair_grants[index].clone();
1806            match grant.phase {
1807                Phase::Pending => {
1808                    if grant.spent >= grant.receipt.request.rounds {
1809                        self.state.repair_grants[index].phase = Phase::Complete;
1810                        self.state.status = RunStatus::Blocked;
1811                        self.state.save()?;
1812                        return Ok(());
1813                    }
1814                    // Reserve before dispatch. A crash can consume an attempt,
1815                    // but can never duplicate it or reset saved fixer history.
1816                    self.state.repair_grants[index].spent += 1;
1817                    self.state.repair_grants[index].phase = Phase::Fixing;
1818                    self.state.save()?;
1819                    match grant.receipt.request.kind {
1820                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821                        Kind::Gate => {
1822                            let failed = self.state.gate.clone();
1823                            let _ = self.gate_fix_round(&winner, &failed).await?;
1824                        }
1825                        Kind::Review => {
1826                            let mut record = self
1827                                .state
1828                                .reviews
1829                                .last()
1830                                .cloned()
1831                                .context("review failure missing")?;
1832                            let round = record.round;
1833                            let step = self.run_fix_round(&winner, &mut record).await?;
1834                            self.state.reviews[round - 1] = record;
1835                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1836                                self.state.repair_grants[index].phase = Phase::Complete;
1837                                self.state.status = RunStatus::Blocked;
1838                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839                                self.state.save()?;
1840                                return Ok(());
1841                            }
1842                            if matches!(step, FixStep::Resource) {
1843                                self.stop_for_resource()?;
1844                                return Ok(());
1845                            }
1846                            self.state.save()?;
1847                        }
1848                    }
1849                    if self.resource_halt()? {
1850                        return Ok(());
1851                    }
1852                }
1853                Phase::Fixing => {
1854                    if grant.receipt.request.kind == Kind::BaseConflict {
1855                        self.repair_base_conflict(false).await?;
1856                    }
1857                    // Interrupted fixers may have left source changes or an
1858                    // unpushed commit. Preserve them before fresh assessment.
1859                    let _coordination = self.coordinate(&winner.worktree).await?;
1860                    let rescued = git::rescue_commit(
1861                        &winner.worktree,
1862                        "magi: preserve interrupted granted repair",
1863                    )
1864                    .await?;
1865                    self.state.note_withheld("repair-grant", &rescued.withheld);
1866                    // Fresh checks must supersede even a same-head failure.
1867                    // Archive rather than erase failure evidence or counters.
1868                    self.state
1869                        .verification_history
1870                        .push(crate::run::VerificationEvidence {
1871                            superseded_by: None,
1872                            archived_at: Some(Timestamp::now()),
1873                            fixes_spent: Some(self.state.gate_fixes.len()),
1874                            gate_head: self.state.gate_head.take(),
1875                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1876                            gate: std::mem::take(&mut self.state.gate),
1877                            e2e: self.state.head_e2e.take(),
1878                        });
1879                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880                        self.state
1881                            .verification_history
1882                            .push(crate::run::VerificationEvidence {
1883                                superseded_by: None,
1884                                archived_at: Some(Timestamp::now()),
1885                                fixes_spent: Some(self.state.gate_fixes.len()),
1886                                gate_head: None,
1887                                gate_ran: false,
1888                                gate: Vec::new(),
1889                                e2e: Some(e2e),
1890                            });
1891                    }
1892                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1893                    self.state.repair_grants[index].head =
1894                        git::rev_parse(&winner.worktree, "HEAD").await?;
1895                    self.state.repair_grants[index].phase = Phase::Reviewing;
1896                    if grant.receipt.request.kind == Kind::BaseConflict {
1897                        self.state.base_sync_recovery.stop = None;
1898                        if let Some(sync) = &mut self.state.base_sync {
1899                            sync.tip = grant
1900                                .receipt
1901                                .request
1902                                .tip
1903                                .clone()
1904                                .context("repair base tip missing")?;
1905                            sync.conflict = None;
1906                            sync.behind = 0;
1907                        }
1908                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909                    }
1910                    self.state.save()?;
1911                }
1912                Phase::Reviewing => {
1913                    self.review_loop_pass().await?;
1914                    if self.resource_halt()? {
1915                        return Ok(());
1916                    }
1917                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918                    let clean = self.state.reviews.len() > grant.review_start
1919                        && self.state.reviews.last().is_some_and(|r| {
1920                            r.head == head
1921                                && r.clean
1922                                && r.expected > 0
1923                                && r.answered == r.expected
1924                                && matches!(
1925                                    r.e2e_status(),
1926                                    E2eStatus::Passed | E2eStatus::NotConfigured
1927                                )
1928                        });
1929                    if !clean {
1930                        // A confirmation review carries no additional fix budget.
1931                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932                            == Kind::Review
1933                            && grant.spent < grant.receipt.request.rounds
1934                            && self.state.reviews.last().is_some_and(|r| {
1935                                r.fix.is_none()
1936                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937                            }) {
1938                            Phase::Pending
1939                        } else {
1940                            Phase::Complete
1941                        };
1942                        self.state.status = RunStatus::Blocked;
1943                        self.state.event(
1944                            "repair-grant",
1945                            "fresh review did not clear the repaired head; no publication",
1946                        );
1947                        self.state.save()?;
1948                        if self.state.repair_grants[index].phase == Phase::Pending {
1949                            continue;
1950                        }
1951                        return Ok(());
1952                    }
1953                    self.state.repair_grants[index].phase = Phase::Gating;
1954                    self.state.save()?;
1955                }
1956                Phase::Gating => {
1957                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958                    self.state.reconcile_verification_head(&head);
1959                    let wait = self.repository_wait();
1960                    if self
1961                        .state
1962                        .reviews
1963                        .last()
1964                        .is_none_or(|r| r.head != head || !r.clean)
1965                    {
1966                        self.state.status = RunStatus::Blocked;
1967                        self.state.repair_grants[index].phase = Phase::Complete;
1968                        self.state.event(
1969                            "repair-grant",
1970                            "head changed since fresh review; repair held",
1971                        );
1972                        self.state.save()?;
1973                        return Ok(());
1974                    }
1975                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976                        return Ok(());
1977                    }
1978                    let outcomes = self.run_gate(&winner).await?;
1979                    if verify_inconclusive(&outcomes) {
1980                        return Ok(());
1981                    }
1982                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983                    if after != head {
1984                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1985                        self.state.repair_grants[index].head = after;
1986                        self.state.repair_grants[index].phase = Phase::Reviewing;
1987                        self.state.status = RunStatus::Blocked;
1988                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989                        self.state.save()?;
1990                        return Ok(());
1991                    }
1992                    let passed = outcomes.iter().all(CommandOutcome::ok);
1993                    // Follow-up grants owe the same guarded checkout delivery
1994                    // as the conflict repair that introduced the new head.
1995                    if passed
1996                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997                    {
1998                        return Ok(());
1999                    }
2000                    self.state.repair_grants[index].succeeded = passed;
2001                    self.state.gate = outcomes;
2002                    self.state.gate_ran = true;
2003                    self.state.status = if passed {
2004                        RunStatus::Gating
2005                    } else {
2006                        RunStatus::Blocked
2007                    };
2008                    self.state.repair_grants[index].phase = if !passed
2009                        && grant.receipt.request.kind == Kind::Gate
2010                        && grant.spent < grant.receipt.request.rounds
2011                    {
2012                        Phase::Pending
2013                    } else {
2014                        Phase::Complete
2015                    };
2016                    if passed && self.state.config.merge.mode == MergeMode::Pr {
2017                        // Persist publication debt with the successful checkpoint.
2018                        self.state.pr_publication_pending = true;
2019                    } else if passed {
2020                        self.state.status = RunStatus::Ready;
2021                    }
2022                    self.state.save()?;
2023                    if !passed {
2024                        continue;
2025                    }
2026                    // Existing PR ownership and push lease stay in place. The
2027                    // normal publisher owes the new reviewed/gated SHA, and
2028                    // land's fresh owner approval remains mandatory.
2029                    if self.state.config.merge.mode != MergeMode::Pr {
2030                        self.state.status = RunStatus::Ready;
2031                        self.state.event(
2032                            "repair-grant",
2033                            "repair verified locally; grant carries no merge authorization",
2034                        );
2035                        self.state.save()?;
2036                        return Ok(());
2037                    }
2038                    return self.merge().await;
2039                }
2040                Phase::Complete => return Ok(()),
2041            }
2042        }
2043    }
2044
2045    async fn execute_graph(&mut self) -> Result<()> {
2046        // Even a terminal run may reenter through an explicit recovery action.
2047        // Restore before prep shortcuts, saved conversations or graph nodes use
2048        // a disposable path; GC may already have removed its original objects.
2049        let archive_home = crate::run::home().canonicalize()?;
2050        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051            let repo = self.state.repo.clone();
2052            let _coordination = self.coordinate(&repo).await?;
2053            ensure_setup_config(&mut self.state, &repo).await?;
2054            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055        }
2056        self.state.configure_recovery_resume()?;
2057        if self
2058            .state
2059            .repair_grants
2060            .last()
2061            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062        {
2063            return self.execute_granted_repair().await;
2064        }
2065        if self
2066            .state
2067            .repair_grants
2068            .last()
2069            .is_some_and(|g| !g.succeeded)
2070        {
2071            self.state.status = RunStatus::Blocked;
2072            self.state.save()?;
2073            return Ok(());
2074        }
2075        // Preservation may invalidate an old empty classification. Resolve it
2076        // before any resume shortcut can reuse a verdict or publish a branch.
2077        self.persist_candidates().await?;
2078        // A saved base stop outranks stage markers, including legacy Voting.
2079        // Recovery never reenters implementation, deliberation or voting.
2080        if self.state.base_sync_blocked() {
2081            self.sync_to_base().await?;
2082            if !self.state.base_sync_blocked()
2083                && self.state.status != RunStatus::AlreadyInBase
2084                && !self.resource_halt()?
2085            {
2086                self.finish_after_tally().await?;
2087            }
2088            return Ok(());
2089        }
2090        if self
2091            .state
2092            .merge_decision
2093            .as_ref()
2094            .is_some_and(|decision| !decision.closed)
2095            && self.state.status != RunStatus::Landing
2096        {
2097            if crate::merge_decision::resumable_hold(&self.state) {
2098                self.state.merge = Some(MergeOutcome {
2099                    mode: MergeMode::Pr,
2100                    ok: false,
2101                    detail: self
2102                        .state
2103                        .merge_decision
2104                        .as_ref()
2105                        .unwrap()
2106                        .target
2107                        .pr
2108                        .clone(),
2109                    empty: false,
2110                });
2111                self.run_land().await?;
2112                self.settle_questions();
2113                return Ok(());
2114            }
2115            bail!(
2116                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117            );
2118        }
2119        if self.state.config.graph.selected_implementer.is_some()
2120            && self.state.status == RunStatus::Blocked
2121            && self.state.tally.is_none()
2122            && self.state.candidate_preservation.is_empty()
2123            && self.state.interrupted_implementations.is_empty()
2124        {
2125            bail!(
2126                "selected implementation is held; explicitly release the owning task to retry it"
2127            );
2128        }
2129
2130        if self.state.merge_repairs.last().is_some_and(|r| {
2131            matches!(
2132                r.phase,
2133                crate::run::MergeRepairPhase::Requested
2134                    | crate::run::MergeRepairPhase::Fixing
2135                    | crate::run::MergeRepairPhase::Reviewing
2136            )
2137        }) {
2138            if self.repair_merge_review().await? {
2139                self.run_land().await?;
2140            }
2141            self.settle_questions();
2142            return Ok(());
2143        }
2144        // Publication already passed review and gate. Resume only that node:
2145        // syncing to a newer base here could change the verified commit.
2146        if self.state.pr_publication_pending {
2147            self.merge().await?;
2148            if self.resource_halt()? {
2149                return Ok(());
2150            }
2151            self.state.save()?;
2152            self.settle_questions();
2153            return Ok(());
2154        }
2155        // A run that already lost its quorum never resumes into the verdict
2156        // machinery: `deliberate` and `vote` would otherwise clobber the
2157        // stalled marker back to Voting and the run would keep going past a
2158        // verdict that is no longer trustworthy. Everything already recorded is
2159        // kept, so the run stays resumable (or foldable) for a human to pick up.
2160        //
2161        // On --resume the run gets one chance to repair itself: the seats a
2162        // rate limit took out are re-asked. If their quota has since reset and
2163        // the quorum is restored, the run picks up and finishes; otherwise it
2164        // stays stale and still-resumable for a later retry. If it does not
2165        // recover, the returned status stays `Stalled` and nothing was
2166        // clobbered (the recovery only mutates entries for the lost seats).
2167        if self.state.status == RunStatus::Stalled {
2168            if self.recover_stall().await? {
2169                self.finish_after_tally().await?;
2170            } else {
2171                // Still below quorum: persist the marker and stay resumable.
2172                self.state.save()?;
2173            }
2174            return Ok(());
2175        }
2176        // A run parked inside `land` - watching CI, mid fix-round, or
2177        // waiting on the owner's merge approval - resumes directly into it,
2178        // never back through `prep`. Everything before `merge` already
2179        // concluded; that is the only way `status` reaches `Landing` in the
2180        // first place. Re-walking `review_loop` first would also be actively
2181        // wrong: its own status recomputation (see its doc) treats any
2182        // clean round as reason to set `status` to `Gating`, which would
2183        // clobber this marker before `merge` ever ran, and this run would
2184        // never find its way back into `land` at all.
2185        if self.state.status == RunStatus::Landing
2186            || crate::handover::landing_checkpoint(&self.state)
2187        {
2188            self.state.land_sync_pending = true;
2189            self.state.save()?;
2190            self.run_land().await?;
2191            // `run_land` may have settled the run right here - CI came back
2192            // green and the PR merged, say - without ever passing back
2193            // through `merge`'s own trailing call. Whatever it left `status`
2194            // as is what this has to read.
2195            self.settle_questions();
2196            return Ok(());
2197        }
2198        // A run parked at `merge` on the owner's word about a withheld PR text
2199        // resumes straight into `merge`. Re-walking the tail would sync to the
2200        // base again (possibly rebasing, possibly spending the fixer) after the
2201        // gate had passed, and then push a tree nobody reviewed or gated.
2202        if self.state.github_text.is_some()
2203            && self.state.merge.is_none()
2204            && self.state.status == RunStatus::Gating
2205        {
2206            self.merge().await?;
2207            if self.resource_halt()? {
2208                return Ok(());
2209            }
2210            self.state.save()?;
2211            return Ok(());
2212        }
2213        if let Err(e) = self.prep().await {
2214            if e.is::<crate::repo_lock::ResourceBlocked>() {
2215                return Err(e);
2216            }
2217            self.state.status = RunStatus::Blocked;
2218            self.state
2219                .prep_failure
2220                .get_or_insert_with(|| format!("{e:#}"));
2221            self.state
2222                .event("prep", format!("preparation stopped: {e:#}"));
2223            self.state.save()?;
2224            return Err(e);
2225        }
2226        if self.park_here()? {
2227            return Ok(());
2228        }
2229        self.advise().await?;
2230        if self.park_here()? {
2231            return Ok(());
2232        }
2233        self.implement().await?;
2234        if self.park_here()? {
2235            return Ok(());
2236        }
2237        // `after_implement` already saved the state and settled any open
2238        // questions when it set this; nothing later in the graph has
2239        // anything to judge.
2240        if self.state.status == RunStatus::VerifiedNoop {
2241            return Ok(());
2242        }
2243        self.judge().await?;
2244        if self.park_here()? {
2245            return Ok(());
2246        }
2247        self.deliberate().await?;
2248        if self.park_here()? {
2249            return Ok(());
2250        }
2251        self.vote().await?;
2252        if self.park_here()? {
2253            return Ok(());
2254        }
2255        self.tally()?;
2256        // A verdict that lost its quorum is not trustworthy: do not review,
2257        // gate, or merge on it. Everything already done is kept, so the run
2258        // stays resumable (or foldable); the human can replace the agent that
2259        // ran out of quota and pick it up.
2260        if self.state.status == RunStatus::Stalled {
2261            // Persist the stalled marker now — the normal end-of-execute save
2262            // below is below this early return, and without it a resumed run
2263            // would reload a pre-tally status and keep going.
2264            self.state.save()?;
2265            return Ok(());
2266        }
2267        self.finish_after_tally().await?;
2268        Ok(())
2269    }
2270
2271    /// A wave could not start its agents because a shared resource stayed busy
2272    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2273    /// resumable. Unstarted seats have no result; completed peers retain
2274    /// their saved candidate state.
2275    fn stop_for_resource(&mut self) -> Result<()> {
2276        // Resource acquisition observes cancellation inside a node. Complete
2277        // the same durable park as every other node boundary before returning.
2278        if self.pause.parked() || self.interrupt.parked() {
2279            self.park_here()?;
2280            return Ok(());
2281        }
2282        if self.state.resource_block.is_some() {
2283            self.state.status = RunStatus::Blocked;
2284        }
2285        self.state.save()
2286    }
2287
2288    /// Stop walking the graph if a node ended on a resource block.
2289    fn resource_halt(&mut self) -> Result<bool> {
2290        if self.state.resource_block.is_none() {
2291            return Ok(false);
2292        }
2293        self.stop_for_resource()?;
2294        Ok(true)
2295    }
2296
2297    /// Park here if asked to, recording it in the run's own timeline.
2298    ///
2299    /// Returns whether the caller should stop walking the graph. The state is
2300    /// saved either way by the node that just finished; this adds the event so
2301    /// the operator's card says why a run that is neither finished nor moving
2302    /// is sitting where it is.
2303    fn park_here(&mut self) -> Result<bool> {
2304        let parking = self.pause.parked() || self.interrupt.parked();
2305        if !parking && self.state.resource_block.is_some() {
2306            self.stop_for_resource()?;
2307            return Ok(true);
2308        }
2309        // Either handle asking is enough - see `Pause`'s own doc for why
2310        // they are never the same one. `interrupt` is checked second so a
2311        // reason it carries is preferred in the message below over a plain
2312        // shutdown park racing it at the same boundary.
2313        if !self.pause.parked() && !self.interrupt.parked() {
2314            return Ok(false);
2315        }
2316        if self.state.parked {
2317            return Ok(true);
2318        }
2319        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320        if let Some(block) = &mut self.state.resource_block
2321            && matches!(
2322                block.cause,
2323                crate::run::ResourceBlockCause::Cancelled { .. }
2324            )
2325        {
2326            block.cause = crate::run::ResourceBlockCause::Cancelled {
2327                park: cause.clone(),
2328            };
2329        }
2330        let why = match cause.as_ref().map(ParkCause::label) {
2331            Some(reason) => format!(
2332                "parked after `{}` ({reason}) — resume to carry on from here",
2333                self.state.status.as_str()
2334            ),
2335            None => format!(
2336                "parked after `{}` — resume to carry on from here",
2337                self.state.status.as_str()
2338            ),
2339        };
2340        self.state.event("park", why);
2341        self.state
2342            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343        self.state.parked = true;
2344        self.state.save()?;
2345        Ok(true)
2346    }
2347
2348    /// Only a new explicit operator approval may retry failed preparation.
2349    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350        if self.state.prep_failure.is_none() {
2351            return Ok(());
2352        }
2353        let failed_at = self
2354            .state
2355            .events
2356            .iter()
2357            .rev()
2358            .find(|event| event.node == "prep")
2359            .map_or(self.state.updated_at, |event| event.at);
2360        anyhow::ensure!(
2361            approval_at > failed_at,
2362            "preparation retry requires a new explicit release after the failure"
2363        );
2364        self.state.prep_failure = None;
2365        self.state.prep_complete = Some(false);
2366        self.state
2367            .event("prep-retry", "operator approved retry of preparation");
2368        Ok(())
2369    }
2370
2371    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2372    pub fn on_pause(&mut self, pause: Pause) {
2373        self.pause = pause;
2374    }
2375
2376    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2377    /// scheduler asking this one run - and no other - to park so a task
2378    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2379    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2380    /// handle.
2381    pub fn watch_interrupt(&mut self, pause: Pause) {
2382        self.interrupt = pause;
2383    }
2384
2385    /// Abandon this run's own open questions, once `status` has actually
2386    /// settled rather than merely paused.
2387    ///
2388    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2389    /// either back up with the candidates, the review round and the seat
2390    /// sessions already on disk, so a question an implementer asked mid-round
2391    /// may still get a real answer read by a real resume. Only the statuses
2392    /// `resumable` excludes are actually final: the run merged, it reached
2393    /// `Ready` with nothing left to do, it failed outright with no
2394    /// established point to continue from, or every candidate agreed, with
2395    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2396    /// every one of those the seat that asked is gone for good, exactly like
2397    /// the run being deleted under `magi run rm` - so the same cleanup
2398    /// applies, worded for what actually happened instead of "the run was
2399    /// deleted".
2400    ///
2401    /// Best-effort and silent on success: called from every place `status`
2402    /// can land on one of those three, including ones a resumed run revisits,
2403    /// so it must cost nothing when there was nothing open to begin with.
2404    fn settle_questions(&mut self) {
2405        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407        }
2408    }
2409
2410    /// The tail of the graph after a trustworthy tally: fold losers, review,
2411    /// gate, merge, and persist.
2412    fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413        Box::pin(self.finish_after_tally_inner())
2414    }
2415
2416    async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417        // Before the losers are folded: their branches are what the graft
2418        // reads from.
2419        self.graft().await?;
2420        if self.resource_halt()? {
2421            return Ok(());
2422        }
2423        self.fold_losers().await?;
2424        // Before review starts, and again right before the gate: a run's
2425        // review rounds can themselves take long enough for the base to move
2426        // a second time, and the gate is the one node whose "green" gets
2427        // acted on.
2428        self.sync_to_base().await?;
2429        if self.hold_base_sync().await? || self.resource_halt()? {
2430            return Ok(());
2431        }
2432        if self.state.status == RunStatus::AlreadyInBase {
2433            return Ok(());
2434        }
2435        self.review_loop().await?;
2436        if self.resource_halt()? {
2437            return Ok(());
2438        }
2439        self.sync_to_base().await?;
2440        if self.hold_base_sync().await? || self.resource_halt()? {
2441            return Ok(());
2442        }
2443        if self.state.status == RunStatus::AlreadyInBase {
2444            return Ok(());
2445        }
2446        self.gate().await?;
2447        self.merge().await?;
2448        if self.resource_halt()? {
2449            return Ok(());
2450        }
2451        self.state.save()?;
2452        Ok(())
2453    }
2454
2455    // ---------------------------------------------------------------- prep
2456
2457    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2458    /// run with the step and its output in the event; it is never swallowed.
2459    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2460    /// resume starts over instead of finding candidates and missing seats.
2461    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462        let result = match ensure_setup_config(&mut self.state, repo).await {
2463            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464            Err(e) => Err(e),
2465        };
2466        if let Err(e) = result {
2467            git::worktree_remove_clean(repo, wt).await.ok();
2468            if in_prep {
2469                self.rollback_prep(repo).await;
2470            }
2471            self.state.status = RunStatus::Blocked;
2472            self.state.event(
2473                "setup",
2474                format!("worktree setup failed in {}: {e:#}", wt.display()),
2475            );
2476            self.state.save()?;
2477            return Err(e);
2478        }
2479        Ok(())
2480    }
2481
2482    /// Undo everything `prep` created, so a resume re-runs it whole.
2483    async fn rollback_prep(&mut self, repo: &Path) {
2484        let root = self.state.worktree_root();
2485        for c in std::mem::take(&mut self.state.candidates) {
2486            if matches!(
2487                git::worktree_remove_clean(repo, &c.worktree).await,
2488                Ok(true)
2489            ) {
2490                git::branch_delete(repo, &c.branch).await.ok();
2491            } else {
2492                self.state.candidates.push(c);
2493            }
2494        }
2495        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496        for j in 1..=seats {
2497            remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2498                .await
2499                .ok();
2500        }
2501        for k in 1..=self.state.config.graph.advisors {
2502            remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2503                .await
2504                .ok();
2505        }
2506    }
2507
2508    async fn prep(&mut self) -> Result<()> {
2509        if let Some(reason) = &self.state.prep_failure {
2510            bail!(
2511                "candidate preparation previously failed: {reason}; correct the source before retrying"
2512            );
2513        }
2514        if self
2515            .state
2516            .prep_complete
2517            .unwrap_or(!self.state.candidates.is_empty())
2518        {
2519            return Ok(());
2520        }
2521        let repo = self.state.repo.clone();
2522        // Includes rollback, config, refs, checkout creation and setup. No
2523        // repository mutation may precede the real shared lock acquisition.
2524        let _coordination = self.coordinate(&repo).await?;
2525        let base = self.state.base_commit.clone();
2526        let plan = self.state.sources.plan(&repo, &base).await?;
2527        self.rollback_prep(&repo).await;
2528        if !self.state.candidates.is_empty() {
2529            bail!(
2530                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531            );
2532        }
2533        self.state.prep_complete = Some(false);
2534        self.state.status = RunStatus::Prep;
2535        self.state.save()?;
2536        let repo = self.state.repo.clone();
2537        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538        let root = self.state.worktree_root();
2539        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541        // The hook is the write-time half of the blindness contract; the
2542        // presentation filter in `blind` is the half that cannot be bypassed.
2543        let hooks_dir = self.state.dir().join("hooks");
2544        if self.state.config.blind.commit_msg_hook {
2545            std::fs::create_dir_all(&hooks_dir)
2546                .with_context(|| format!("create {}", hooks_dir.display()))?;
2547            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548            let path = hooks_dir.join("commit-msg");
2549            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550            make_executable(&path)?;
2551            // Ref-counted rather than a plain idempotent set: with more than
2552            // one run able to be in flight in the same repository at once
2553            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2554            // true?" check cannot tell "another run of mine still needs
2555            // this" from "nobody does", and the run that happens to finish
2556            // first would disable the hook out from under a sibling still
2557            // relying on it.
2558            git::acquire_worktree_config(&repo).await?;
2559            self.state.enabled_worktree_config = true;
2560        }
2561
2562        // `[worktree] setup` hides its products per worktree, which needs the
2563        // same per-worktree config the hook does. Held until fold, like it.
2564        ensure_setup_config(&mut self.state, &repo).await?;
2565
2566        for (index, (spec, label)) in self
2567            .roles
2568            .implementers
2569            .clone()
2570            .into_iter()
2571            .zip(labels)
2572            .enumerate()
2573        {
2574            let branch = self.state.branch_for(label);
2575            let worktree = root.join(format!("cand-{label}"));
2576            // Persist intent before Git can leave a partially created tree.
2577            // Candidate registration follows setup, so it cannot inventory
2578            // crashes or refusals during creation itself.
2579            self.state.event(
2580                "prep-intent",
2581                serde_json::json!({
2582                    "branch": branch,
2583                    "worktree": worktree,
2584                    "start": start,
2585                    "picks": plan.picks,
2586                })
2587                .to_string(),
2588            );
2589            self.state.save()?;
2590            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591            if self.state.config.blind.commit_msg_hook {
2592                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593            }
2594            git::local_exclude(&worktree, "/.magi/").await?;
2595            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596                // Nothing of this prep may survive: a resume must start over,
2597                // not find half the candidates and skip the rest.
2598                git::worktree_remove_clean(&repo, &worktree).await.ok();
2599                git::branch_delete(&repo, &branch).await.ok();
2600                self.rollback_prep(&repo).await;
2601                self.state.status = RunStatus::Blocked;
2602                self.state
2603                    .event("prep", format!("worktree setup failed: {e:#}"));
2604                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605                self.state.save()?;
2606                return Err(e);
2607            }
2608            self.state.candidates.push(Candidate {
2609                index,
2610                label,
2611                agent: spec.id.clone(),
2612                branch,
2613                worktree,
2614                summary: String::new(),
2615                stat: String::new(),
2616                files: 0,
2617                commits: 0,
2618                empty: false,
2619                failed: None,
2620                verified_noop: None,
2621                duration_ms: 0,
2622                folded: false,
2623            });
2624            self.state.save()?;
2625            for pick in &plan.picks {
2626                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627                {
2628                    self.state.status = RunStatus::Blocked;
2629                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630                    self.state
2631                        .event("prep", format!("cannot apply referenced commit: {e}"));
2632                    self.state.save()?;
2633                    return Err(e);
2634                }
2635            }
2636        }
2637
2638        for j in 1..=self.roles.judges.len() {
2639            let wt = root.join(format!("judge-{j}"));
2640            if !wt.exists() {
2641                git::worktree_add_detached(&repo, &wt, &base).await?;
2642                self.setup_seat_worktree(&repo, &wt, true).await?;
2643            }
2644        }
2645
2646        // Disposable, detached checkouts for the design-deliberation stage's
2647        // advisor seats — the same shape as the judges' above, at the same
2648        // base commit, since advisors also only ever read. Sized off the
2649        // configured count directly rather than a resolved roster: unlike
2650        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2651        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2652        // `prep` has no `ResolvedRoles` field to read a count from here.
2653        if self.state.config.graph.advise {
2654            for k in 1..=self.state.config.graph.advisors {
2655                let wt = root.join(format!("advisor-{k}"));
2656                if !wt.exists() {
2657                    git::worktree_add_detached(&repo, &wt, &base).await?;
2658                    self.setup_seat_worktree(&repo, &wt, true).await?;
2659                }
2660            }
2661        }
2662
2663        // A judge cannot tell it is looking at its own patch — the seats keep
2664        // separate conversations — but a panel that shares agents with the
2665        // field is less independent than it looks, and that is worth saying out
2666        // loud once per run rather than leaving it in the config.
2667        let authors: Vec<&str> = self
2668            .roles
2669            .implementers
2670            .iter()
2671            .map(|a| a.id.as_str())
2672            .collect();
2673        let overlap: Vec<String> = self
2674            .roles
2675            .judges
2676            .iter()
2677            .enumerate()
2678            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680            .collect();
2681        if !overlap.is_empty() {
2682            let note = format!(
2683                "{} also authored a candidate; blind, but the panel is less \
2684                 independent than {} distinct agents would be",
2685                overlap.join(", "),
2686                self.roles.judges.len()
2687            );
2688            self.state.event("prep", note);
2689        }
2690
2691        self.state.event(
2692            "prep",
2693            format!(
2694                "{} candidates, {} judges, base {} ({})",
2695                self.state.candidates.len(),
2696                self.roles.judges.len(),
2697                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698                self.state.base_branch
2699            ),
2700        );
2701        self.state.prep_complete = Some(true);
2702        self.state.status = RunStatus::Implementing;
2703        self.state.save()?;
2704        Ok(())
2705    }
2706
2707    // -------------------------------------------------------------- advise
2708
2709    /// The design-deliberation stage: independent, read-only advisor seats
2710    /// each sketch a design before any implementer touches the repository,
2711    /// and (when at least one produced a usable proposal) a synthesis seat
2712    /// blends them into a brief `implement` carries in every candidate's
2713    /// prompt.
2714    ///
2715    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2716    /// advisors` is the proposal count. Everything here is best-effort and
2717    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2718    /// that cannot reach quota, or a synthesis seat that produced nothing
2719    /// usable all leave `implement` exactly as it was before this stage
2720    /// existed — the task instruction alone — rather than failing the whole
2721    /// competition over an enrichment stage. Every outcome is still recorded
2722    /// as an event, so a run that got nothing from this stage says why.
2723    ///
2724    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2725    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2726    /// resumed run whose stage failed would re-run it, and re-spend the
2727    /// agent calls, on every reentry before `implement`.
2728    ///
2729    /// Also skipped once any candidate shows implementation progress — the
2730    /// exact predicate `implement` itself uses to decide a candidate is no
2731    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2732    /// is not enough: a run created by an older binary that predates this
2733    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2734    /// an already-`Implementing`-or-later run under this build would
2735    /// otherwise walk straight back through `prep` (a no-op once candidates
2736    /// exist) into this node and spawn every advisor seat against worktrees
2737    /// `prep` never recreated — after implementation has already started,
2738    /// which is exactly the invariant this stage exists to guarantee.
2739    async fn advise(&mut self) -> Result<()> {
2740        let implement_untouched = self
2741            .state
2742            .candidates
2743            .iter()
2744            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745        if !self.state.config.graph.advise || self.state.advise_attempted {
2746            return Ok(());
2747        }
2748        if !implement_untouched {
2749            self.state.event(
2750                "advise",
2751                "skipping the design-deliberation stage: at least one \
2752                 candidate already shows implementation progress, so this \
2753                 run is past the point the stage exists to run before"
2754                    .to_owned(),
2755            );
2756            self.state.advise_attempted = true;
2757            self.state.save()?;
2758            return Ok(());
2759        }
2760        let run_id = self.state.id.clone();
2761        let prompts = self.state.config.prompts.clone();
2762        let instruction = self.state.instruction.clone();
2763        let language = self.state.config.graph.language.clone();
2764        let root = self.state.worktree_root();
2765        let n = self.state.config.graph.advisors;
2766        let where_recorded = self.state.dir().join("run.json");
2767
2768        let seats = match self.state.config.advisors() {
2769            Ok(seats) if !seats.is_empty() => seats,
2770            Ok(_) => {
2771                self.state.event(
2772                    "advise",
2773                    format!(
2774                        "[graph] advisors is 0; skipping the design-deliberation \
2775                         stage and continuing without a synthesis brief (see {})",
2776                        where_recorded.display()
2777                    ),
2778                );
2779                self.state.advise_attempted = true;
2780                self.state.save()?;
2781                return Ok(());
2782            }
2783            Err(e) => {
2784                self.state.event(
2785                    "advise",
2786                    format!(
2787                        "could not resolve advisor seats ({e:#}); continuing \
2788                         without a design-deliberation brief (see {})",
2789                        where_recorded.display()
2790                    ),
2791                );
2792                self.state.advise_attempted = true;
2793                self.state.save()?;
2794                return Ok(());
2795            }
2796        };
2797
2798        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799        let artifacts = agent::artifacts_dir(&self.state.dir());
2800        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802        let mut jobs = Vec::new();
2803        for (i, spec) in seats.iter().cloned().enumerate() {
2804            let seat_key = format!("advisor-{}", i + 1);
2805            let seat = self.seat(&seat_key, &spec.id);
2806            jobs.push(SeatJob {
2807                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808                spec,
2809                seat,
2810                cwd: worktrees[i % worktrees.len()].clone(),
2811                timeout,
2812                allow_write: false,
2813                sessions: false,
2814                artifacts: artifacts.clone(),
2815                stem: seat_key,
2816                handover: None,
2817            });
2818        }
2819
2820        self.state.event(
2821            "advise",
2822            format!(
2823                "{} advisor seat(s) sketching a design in parallel",
2824                jobs.len()
2825            ),
2826        );
2827        let mut quota_losses = Vec::new();
2828        let cache = self.state.config.cache_dir();
2829        let ctx = WaveCtx {
2830            carry_seats: false,
2831            pauses: Some((&self.pause, &self.interrupt)),
2832            run: &run_id,
2833            node: "advise",
2834            prompts: &prompts,
2835            cache: cache.as_deref(),
2836            round: None,
2837        };
2838        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839        let results = ask_json_wave::<Proposal>(
2840            jobs,
2841            Arc::clone(&self.sem),
2842            self.state.config.graph.retries,
2843            &advisor_roster,
2844            &ctx,
2845            &mut quota_losses,
2846            &mut self.state,
2847            &|p: &Proposal| p.validate(),
2848        )
2849        .await;
2850        if self.state.resource_block.is_some() {
2851            return self.stop_for_resource();
2852        }
2853        self.state.quota.extend(quota_losses);
2854
2855        let mut records = Vec::with_capacity(results.len());
2856        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857            let agent_id = seat.agent.clone();
2858            self.state.seats.insert(seat.key.clone(), seat);
2859            match res {
2860                Ok((proposal, out)) => {
2861                    self.state
2862                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2863                    records.push(advise::AdvisorRecord::proposed(
2864                        i + 1,
2865                        agent_id,
2866                        proposal,
2867                        out.duration_ms,
2868                    ));
2869                }
2870                Err(e) => {
2871                    self.state.event(
2872                        "advise",
2873                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874                    );
2875                    records.push(advise::AdvisorRecord::failed(
2876                        i + 1,
2877                        agent_id,
2878                        e.to_string(),
2879                    ));
2880                }
2881            }
2882        }
2883
2884        let mut advice = advise::Advice {
2885            records,
2886            synthesis: None,
2887        };
2888        if advice.proposals().is_empty() {
2889            self.state.event(
2890                "advise",
2891                "no advisor produced a usable proposal; continuing without a \
2892                 synthesis brief"
2893                    .to_owned(),
2894            );
2895        } else {
2896            match self
2897                .synthesize_brief(
2898                    &advice,
2899                    &instruction,
2900                    &language,
2901                    &worktrees[0],
2902                    &artifacts,
2903                    &run_id,
2904                    &prompts,
2905                    cache.as_deref(),
2906                )
2907                .await
2908            {
2909                Ok(Some(text)) => {
2910                    self.state.event(
2911                        "advise",
2912                        "synthesized a design brief for the implementer".to_owned(),
2913                    );
2914                    advice.synthesis = Some(text);
2915                }
2916                Ok(None) => {
2917                    self.state.event(
2918                        "advise",
2919                        "the synthesis seat produced nothing usable; continuing \
2920                         without a design brief"
2921                            .to_owned(),
2922                    );
2923                }
2924                Err(e) => {
2925                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2926                        return Err(e);
2927                    }
2928                    self.state.event(
2929                        "advise",
2930                        format!("could not synthesize a design brief: {e:#}"),
2931                    );
2932                }
2933            }
2934        }
2935        advise::apply_reflection(&mut advice);
2936
2937        self.state.advice = Some(advice);
2938        self.state.advise_attempted = true;
2939        self.state.save()?;
2940        Ok(())
2941    }
2942
2943    /// The synthesis seat: reads every advisor's proposal and blends them
2944    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2945    /// out of [`Runner::advise`] only for readability — it is not called
2946    /// anywhere else.
2947    ///
2948    /// Picked the same way [`crate::talk`]'s standing conversation and
2949    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2950    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2951    /// order (a claude seat, else the first runnable agent in roster order)
2952    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2953    /// in [`crate::config`] for why a dedicated field exists here at all.
2954    #[allow(clippy::too_many_arguments)]
2955    async fn synthesize_brief(
2956        &mut self,
2957        advice: &advise::Advice,
2958        instruction: &str,
2959        language: &str,
2960        cwd: &Path,
2961        artifacts: &Path,
2962        run_id: &str,
2963        prompts: &Prompts,
2964        cache: Option<&Path>,
2965    ) -> Result<Option<String>> {
2966        let chain = agent::pick_chain(
2967            &self.state.config.agents,
2968            self.state.config.roles.synthesizer.as_ref(),
2969            &agent::installed,
2970            "synthesizer",
2971        )?;
2972        let proposals = advice.proposals();
2973        let mut prompt = prompt::with_overlay(
2974            prompt::synthesize_brief(instruction, &proposals, language),
2975            prompts.overlay("advise"),
2976        );
2977        if cache.is_some() {
2978            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2979            // below — see `prompt::build_cache_note`'s doc for why telling a
2980            // read-only seat to build through the shared cache is exactly how
2981            // a sandbox's write refusal gets misread as a defect.
2982            prompt.push('\n');
2983            prompt.push_str(&prompt::build_cache_note("advise", false));
2984        }
2985        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986        // Each id is tried once, in order; a quota hit, error or unusable
2987        // answer moves to the next. The seat is single-turn (`sessions:
2988        // false`) and the prompt is the whole context, so a fallback agent
2989        // needs nothing carried over.
2990        let mut last = None;
2991        for (n, spec) in chain.iter().enumerate() {
2992            if n > 0 {
2993                self.state
2994                    .event("advise", format!("synthesis falling back to {}", spec.id));
2995            }
2996            let mut seat = self.seat("advise-synthesis", &spec.id);
2997            let wait = self.repository_wait();
2998            let outcome = wait
2999                .scope(agent::invoke(
3000                    spec,
3001                    &mut seat,
3002                    &Invocation {
3003                        coordination: self.state.config.coordination.as_ref(),
3004                        fetched_base: None,
3005                        cwd,
3006                        prompt: &prompt,
3007                        timeout,
3008                        allow_write: false,
3009                        sessions: false,
3010                        artifacts,
3011                        stem: &if n == 0 {
3012                            "advise-synthesis".to_owned()
3013                        } else {
3014                            format!("advise-synthesis-{}", spec.id)
3015                        },
3016                        run: run_id,
3017                        node: "advise",
3018                        cache_dir: None,
3019                        attachments: &[],
3020                        writable: &[],
3021                    },
3022                ))
3023                .await;
3024            if outcome.is_ok() {
3025                self.state.seats.insert(seat.key.clone(), seat);
3026            }
3027            let advance = agent::chain_advances(&outcome);
3028            last = Some(outcome);
3029            if !advance {
3030                break;
3031            }
3032        }
3033        // Exhausted: the last attempt's result is what a single failed seat
3034        // would have produced.
3035        let out = last.expect("a chain holds at least one agent")?;
3036        if !out.usable() {
3037            return Ok(None);
3038        }
3039        let text =
3040            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041        Ok((!text.trim().is_empty()).then_some(text))
3042    }
3043
3044    // ----------------------------------------------------------- implement
3045
3046    async fn implement(&mut self) -> Result<()> {
3047        self.persist_candidates().await?;
3048        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3049        // agent files with `magi task add` name the run that paid for it. The
3050        // prompt overlay is cloned alongside it because the waves borrow it
3051        // while `self` is mutably borrowed by the node's own bookkeeping.
3052        let run_id = self.state.id.clone();
3053        let prompts = self.state.config.prompts.clone();
3054        let todo: Vec<usize> = self
3055            .state
3056            .candidates
3057            .iter()
3058            .enumerate()
3059            .filter(|(_, c)| {
3060                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061                    && c.failed.is_none()
3062                    && !c.empty
3063            })
3064            .map(|(i, _)| i)
3065            .collect();
3066        if todo.is_empty() {
3067            return self.after_implement();
3068        }
3069        self.state.status = RunStatus::Implementing;
3070
3071        let language = self.state.config.graph.language.clone();
3072        let github_language = self.state.config.graph.github_language.clone();
3073        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074        let sessions = self.state.config.graph.sessions;
3075        let artifacts = agent::artifacts_dir(&self.state.dir());
3076        // The design-deliberation stage's blended brief, when `advise` found
3077        // one — carried into every implementer's prompt the same way
3078        // regardless of which candidate it is.
3079        let brief = self
3080            .state
3081            .advice
3082            .as_ref()
3083            .and_then(|a| a.synthesis.as_deref())
3084            .map(str::to_owned);
3085        let attachments = self.state.attachments.clone();
3086
3087        let mut jobs = Vec::new();
3088        for &i in &todo {
3089            let (index, label, worktree) = {
3090                let c = &self.state.candidates[i];
3091                (c.index, c.label, c.worktree.clone())
3092            };
3093            let spec = self.roles.implementers[index].clone();
3094            let seat_key = format!("impl-{label}");
3095            let seat = self.seat(&seat_key, &spec.id);
3096            let instruction = seeded_instruction(&self.state);
3097            jobs.push(SeatJob {
3098                spec,
3099                seat,
3100                prompt: prompt::implement_in(
3101                    &instruction,
3102                    &worktree.to_string_lossy(),
3103                    &language,
3104                    &github_language,
3105                    brief.as_deref(),
3106                    &attachments,
3107                ),
3108                cwd: worktree,
3109                timeout,
3110                allow_write: true,
3111                sessions,
3112                artifacts: artifacts.clone(),
3113                stem: format!("impl-{label}"),
3114                handover: None,
3115            });
3116        }
3117
3118        self.state.event(
3119            "implement",
3120            format!("{} candidates in parallel", jobs.len()),
3121        );
3122        // Kept so a seat whose CLI hung up can be asked again from the same
3123        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3124        // can update a seat's own entry once a fallback agent takes it over —
3125        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3126        // whichever agent actually answered, not the one that quota'd out.
3127        let mut sent = jobs.clone();
3128        let cache = self.state.config.cache_dir();
3129        let ctx = WaveCtx {
3130            carry_seats: false,
3131            pauses: Some((&self.pause, &self.interrupt)),
3132            run: &run_id,
3133            node: "implement",
3134            prompts: &prompts,
3135            cache: cache.as_deref(),
3136            round: None,
3137        };
3138        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139        if !wave_blocked(&results) {
3140            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141                .await;
3142            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143                .await?;
3144            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145                .await;
3146        }
3147        // Do not finalize failed candidates when the entire wave needs machine
3148        // recovery: leaving them pending resumes this node with its work intact.
3149        if !results.is_empty()
3150            && results
3151                .iter()
3152                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153        {
3154            let losses: Vec<_> = results
3155                .iter()
3156                .map(|(_, seat, out)| {
3157                    let AgentOutcome::Quota(o) = out else {
3158                        unreachable!()
3159                    };
3160                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161                })
3162                .collect();
3163            self.state.quota.extend(losses.clone());
3164            for (_, seat, _) in results {
3165                self.state.seats.insert(seat.key.clone(), seat);
3166            }
3167            for &i in &todo {
3168                let label = self.state.candidates[i].label;
3169                self.state.interrupted_implementations.insert(label);
3170                self.state
3171                    .candidate_preservation
3172                    .insert(label, "machine outage awaiting persistence".into());
3173            }
3174            // Preserve partial edits too, while keeping every interrupted seat
3175            // pending even when its tree already contains an inherited patch.
3176            block_machine_outage(&mut self.state, "implement", &losses);
3177            self.state.save()?;
3178            self.persist_candidates().await?;
3179            for &i in &todo {
3180                self.state
3181                    .interrupted_implementations
3182                    .insert(self.state.candidates[i].label);
3183            }
3184            return self.stop_for_resource();
3185        }
3186        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189            && results
3190                .iter()
3191                .any(|(_, _, out)| FailClass::of(out).is_some());
3192        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194                // Executed turns interrupted by disk pressure already carry
3195                // a preservation barrier from wave. Admission-only waits do
3196                // not. Keep the interrupted seat's session for recovery too.
3197                if self
3198                    .state
3199                    .candidate_preservation
3200                    .contains_key(&self.state.candidates[i].label)
3201                {
3202                    self.state.seats.insert(seat.key.clone(), seat);
3203                }
3204                continue;
3205            }
3206            let seat_key = seat.key.clone();
3207            // A quota fallback (`resume_seat_handovers`) may have handed this
3208            // seat to a different agent than the one `prep` recorded on the
3209            // candidate; the stats tables and any later fixer-defaults-to-
3210            // winner's-author lookup must credit whoever actually answered —
3211            // unless every fallback also quota'd out, in which case nobody
3212            // actually answered and crediting the last agent tried would
3213            // erase every earlier agent's own quota loss from the stats
3214            // tables instead of just this one seat's.
3215            let agent = seat.agent.clone();
3216            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217            self.state.seats.insert(seat.key.clone(), seat);
3218            let label = self.state.candidates[i].label;
3219            self.state.interrupted_implementations.remove(&label);
3220
3221            let (summary, duration, failed, verified_claim) = match out {
3222                AgentOutcome::Ok(o) => {
3223                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224                    let failed = (!o.usable()).then(|| {
3225                        if o.timed_out {
3226                            "agent timed out".to_owned()
3227                        } else {
3228                            format!(
3229                                "agent exited with {}",
3230                                crate::notices::exit_status(o.exit_code)
3231                            )
3232                        }
3233                    });
3234                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235                    (text, o.duration_ms, failed, verified_claim)
3236                }
3237                // Left un-resumed by `resume_undelivered` (a dirty tree
3238                // already rescues the work, or there was no session left to
3239                // resume into) — reported like the ordinary failure it is,
3240                // never as if `o.text` (the CLI's raw error JSON) were an
3241                // answer.
3242                AgentOutcome::Dropped(o) => {
3243                    let why = o
3244                        .dropped
3245                        .as_ref()
3246                        .map(|d| d.why.as_str())
3247                        .unwrap_or("the CLI ended the stream without delivering its answer");
3248                    (
3249                        String::new(),
3250                        o.duration_ms,
3251                        Some(format!("the CLI dropped the stream ({why})")),
3252                        None,
3253                    )
3254                }
3255                AgentOutcome::Quota(o) => {
3256                    self.state.quota.push(QuotaLoss::from_report(
3257                        seat_key,
3258                        "implement",
3259                        o.quota.as_ref(),
3260                    ));
3261                    (
3262                        String::new(),
3263                        o.duration_ms,
3264                        Some("rate limited (quota); produced no change".to_owned()),
3265                        None,
3266                    )
3267                }
3268                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269                    (String::new(), 0, Some(e), None)
3270                }
3271            };
3272
3273            // Save every completed turn before any fallible Git operation. A failed
3274            // rescue must not erase the reply or prevent collection of other seats.
3275            let c = &mut self.state.candidates[i];
3276            if !exhausted_the_fallback_chain {
3277                c.agent = agent;
3278            }
3279            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280            c.duration_ms = duration;
3281            c.failed = failed;
3282            c.verified_noop = verified_claim;
3283            self.state
3284                .candidate_preservation
3285                .insert(label, "awaiting persistence".to_owned());
3286        }
3287        self.state.save()?;
3288        self.persist_candidates().await?;
3289
3290        if resource_blocked {
3291            return self.stop_for_resource();
3292        }
3293        if selected_failed {
3294            self.state.status = RunStatus::Blocked;
3295            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296            self.state.save()?;
3297            bail!(
3298                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299            );
3300        }
3301        self.after_implement()
3302    }
3303
3304    /// Retry only persistence after a completed implementation turn.
3305    async fn persist_candidates(&mut self) -> Result<()> {
3306        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307        for label in labels {
3308            let i = self
3309                .state
3310                .candidates
3311                .iter()
3312                .position(|c| c.label == label)
3313                .context("pending candidate is missing")?;
3314            let c = self.state.candidates[i].clone();
3315            let interrupted = self.state.interrupted_implementations.contains(&label);
3316            let result = async {
3317                let wait = crate::repo_lock::WaitPolicy::new(
3318                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319                    || false,
3320                );
3321                let _coordination =
3322                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323                // Best effort evidence requires no index writes. A successful snapshot
3324                // never authorizes removal of a tree whose rescue failed.
3325                if let Err(e) = git::snapshot_uncommitted(
3326                    &c.worktree,
3327                    &self
3328                        .state
3329                        .dir()
3330                        .join("artifacts")
3331                        .join(format!("candidate-{label}-recovery")),
3332                )
3333                .await
3334                {
3335                    self.state.event(
3336                        "preserve",
3337                        format!("candidate {label} snapshot incomplete: {e:#}"),
3338                    );
3339                }
3340                let rescued = git::rescue_commit(
3341                    &c.worktree,
3342                    &format!("magi: candidate {label} (uncommitted work)"),
3343                )
3344                .await?;
3345                self.state.note_withheld("implement", &rescued.withheld);
3346                let base = &self.state.base_commit;
3347                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352                if c.empty && commits > 0 && !patch.trim().is_empty()
3353                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355                        || self.state.tally.is_some())
3356                {
3357                    // Keep the old classification and pending marker so every
3358                    // retry stops too. Retain decisions as recovery evidence;
3359                    // only a supported recovery operation may replace them.
3360                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361                }
3362                let c = &mut self.state.candidates[i];
3363                c.commits = commits;
3364                c.stat = stat;
3365                c.files = files;
3366                let no_changes = commits == 0 || patch.trim().is_empty();
3367                c.empty = no_changes && !interrupted;
3368                if interrupted && no_changes {
3369                    c.failed = None;
3370                    c.verified_noop = None;
3371                }
3372                if !c.empty {
3373                    c.failed = None;
3374                    c.verified_noop = None;
3375                }
3376                self.state.event(
3377                    "preserve",
3378                    format!("candidate {label}: {files} files, {commits} commits"),
3379                );
3380                Ok::<_, anyhow::Error>(())
3381            }
3382            .await;
3383            match result {
3384                Ok(()) => {
3385                    self.state.candidate_preservation.remove(&label);
3386                    if !interrupted || self.state.candidates[i].files > 0 {
3387                        self.state.interrupted_implementations.remove(&label);
3388                    }
3389                }
3390                Err(e) => {
3391                    let reason = format!("candidate {label} preservation failed: {e:#}");
3392                    self.state.event("preserve", &reason);
3393                    self.state.candidate_preservation.insert(label, reason);
3394                }
3395            }
3396            if self.state.candidate_preservation.is_empty()
3397                && self.state.status == RunStatus::Blocked
3398                && self.state.candidates.iter().all(|c| c.failed.is_none())
3399            {
3400                // Persist the recovered phase with the cleared pending marker;
3401                // a crash between saves must not turn this into a selected-seat
3402                // failure hold on the next resume.
3403                self.state.status = RunStatus::Implementing;
3404                self.state.event(
3405                    "preserve",
3406                    "candidate persistence recovered; implementation can continue",
3407                );
3408            }
3409            self.state.save()?;
3410        }
3411        if !self.state.candidate_preservation.is_empty() {
3412            self.state.status = RunStatus::Blocked;
3413            self.state.save()?;
3414            bail!(
3415                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416            );
3417        }
3418        Ok(())
3419    }
3420
3421    /// Ask again, once, for work a CLI did and then failed to hand over.
3422    ///
3423    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3424    /// status with an empty response and a usage report showing output tokens,
3425    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3426    /// seven minutes and 14,267 output tokens that arrived as an empty
3427    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3428    ///
3429    /// Two conditions, and both matter:
3430    ///
3431    /// - **Only when the tree is untouched.** Often the agent has already
3432    ///   written its files and only the closing message was lost; the rescue
3433    ///   commit below picks that up and there is nothing to ask for. Re-asking
3434    ///   then would pay for a second implementation of work already on disk.
3435    /// - **Once.** A CLI that drops one stream can drop the next, and this
3436    ///   node is the most expensive in the graph.
3437    ///
3438    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3439    /// dropped reply still carried its `conversation_id`, so the seat is asked
3440    /// to finish what it was doing rather than sent the whole task again. It
3441    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3442    /// node's - for the same reason a re-ranked judge does: restating finished
3443    /// work is not the work.
3444    ///
3445    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3446    /// same way until it resets, while an abandoned conversation is still
3447    /// there to be picked up.
3448    async fn resume_undelivered(
3449        &mut self,
3450        results: &mut [(usize, SeatState, AgentOutcome)],
3451        sent: &[SeatJob],
3452        prompts: &Prompts,
3453        run_id: &str,
3454    ) {
3455        for (wi, seat, out) in results.iter_mut() {
3456            let Some(dropped) = (match &*out {
3457                AgentOutcome::Dropped(o) => o.dropped.clone(),
3458                _ => None,
3459            }) else {
3460                continue;
3461            };
3462            let Some(job) = sent.get(*wi) else { continue };
3463            // Already on disk? Then only the closing message was lost.
3464            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465                self.state.event(
3466                    "implement",
3467                    format!(
3468                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469                         work is in the tree",
3470                        seat.key, dropped.output_tokens, dropped.why
3471                    ),
3472                );
3473                continue;
3474            }
3475            // The re-ask only makes sense as a resume: `resume_after_drop`
3476            // says nothing about the task, trusting the seat to still hold it.
3477            // Without a session to resume — sessions disabled, or this CLI's
3478            // drop shape happened not to carry a session id — that prompt
3479            // would open a brand-new conversation with no context at all,
3480            // which is worse than leaving this as the ordinary failure it
3481            // already is.
3482            if !has_context(&job.spec, seat, job.sessions) {
3483                self.state.event(
3484                    "implement",
3485                    format!(
3486                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487                         is no session left to resume",
3488                        seat.key, dropped.output_tokens, dropped.why
3489                    ),
3490                );
3491                continue;
3492            }
3493            self.state.event(
3494                "implement",
3495                format!(
3496                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497                     conversation",
3498                    seat.key, dropped.output_tokens, dropped.why
3499                ),
3500            );
3501            let mut retry = job.clone();
3502            retry.seat = seat.clone();
3503            retry.prompt = prompt::resume_after_drop(&dropped.why);
3504            retry.timeout = retry_budget(job.timeout, true);
3505            retry.stem = format!("{}-resume", job.stem);
3506            let cache = self.state.config.cache_dir();
3507            let ctx = WaveCtx {
3508                carry_seats: false,
3509                pauses: Some((&self.pause, &self.interrupt)),
3510                run: run_id,
3511                node: "implement",
3512                prompts,
3513                cache: cache.as_deref(),
3514                round: None,
3515            };
3516            let (resumed_seat, resumed) =
3517                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518            *seat = resumed_seat;
3519            *out = resumed;
3520        }
3521    }
3522
3523    /// Fall an implement seat through to the next untried agent in the
3524    /// implementer roster when it lost to quota — or, since the handover was
3525    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3526    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3527    /// quota path itself being unchanged — instead of leaving the
3528    /// seat's loss final the moment one agent's account runs dry.
3529    ///
3530    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3531    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3532    /// truncates to the single slot rotation picked, so a solo task whose one
3533    /// implementer hits quota mid-run used to have nothing else to try. This
3534    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3535    /// unrotated roster — which is the only place the *other* candidates in
3536    /// the machine's roster still exist once `implementers` has been cut down
3537    /// to size.
3538    ///
3539    /// Walks forward from just past the seat's own original position in the
3540    /// roster, never wrapping back to the front: a later candidate slot (say
3541    /// `beta`, the roster's second entry) must fall through to the *next*
3542    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3543    /// almost certainly a different candidate's own agent already — and once
3544    /// the roster's tail is exhausted there is nothing left to fall through
3545    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3546    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3547    /// let this retry that id forever. The loop keeps falling through until
3548    /// an attempt lands something other than `Quota` or the roster's tail
3549    /// runs out of untried ids, at which point the seat is left exactly as
3550    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3551    /// `QuotaLoss` recorded, the candidate failed/empty.
3552    ///
3553    /// `sent` is taken mutably and updated with the fallback agent's spec:
3554    /// `resume_unconfirmed_commands`, which runs after this and also reads
3555    /// `sent`, must see whichever agent actually ended up answering the seat
3556    /// — reading the stale, original spec there would check session
3557    /// eligibility against the wrong CLI and could hand a fallback agent's
3558    /// session id to the agent that just lost the seat to quota.
3559    ///
3560    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3561    /// — `self.seat` only reuses state when the agent id is unchanged, so
3562    /// handing it a different id already gets this for free. Reusing the old
3563    /// seat would resume a different CLI's session as if it were a
3564    /// continuation of this one.
3565    ///
3566    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3567    /// a quota loss cuts an agent off mid-turn, so anything already in the
3568    /// tree is unfinished work, not a completed candidate a re-ask would pay
3569    /// for twice. A dirty tree is rescued into a commit first (the same
3570    /// neutral-identity rescue `implement`'s own outcome loop gives every
3571    /// candidate) so the next agent starts clean.
3572    ///
3573    /// The new agent gets the implementer's full prompt and full
3574    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3575    /// it has no session and no context, and is implementing the task from
3576    /// nothing, unlike a resumed drop which is only restating work already
3577    /// done.
3578    ///
3579    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3580    /// `implement` event, never into `self.state.quota` — that is what
3581    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3582    /// go unspent, and a seat that ultimately recovered on its second or
3583    /// third agent is not the stalled panel that check exists to catch. Only
3584    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3585    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3586    /// outcome loop already has — this helper never pushes to it itself.
3587    async fn resume_seat_handovers(
3588        &mut self,
3589        results: &mut [(usize, SeatState, AgentOutcome)],
3590        sent: &mut [SeatJob],
3591        prompts: &Prompts,
3592        run_id: &str,
3593    ) -> Result<()> {
3594        let instruction = seeded_instruction(&self.state);
3595        let language = self.state.config.graph.language.clone();
3596        let github_language = self.state.config.graph.github_language.clone();
3597        let brief = self
3598            .state
3599            .advice
3600            .as_ref()
3601            .and_then(|a| a.synthesis.as_deref())
3602            .map(str::to_owned);
3603        let attachments = self.state.attachments.clone();
3604        for (wi, seat, out) in results.iter_mut() {
3605            // Who holds the other candidate seats of this wave right now
3606            // (earlier handovers already written back to `sent`).
3607            let others: BTreeSet<String> = sent
3608                .iter()
3609                .enumerate()
3610                .filter(|(j, _)| j != wi)
3611                .map(|(_, j)| j.spec.id.clone())
3612                .collect();
3613            let Some(job) = sent.get_mut(*wi) else {
3614                continue;
3615            };
3616            // Where the seat's own original agent sits in the roster — the
3617            // fallback walk starts just past here, never at the front, so a
3618            // later candidate slot's quota loss does not fall back onto an
3619            // earlier slot's own agent.
3620            let start = self
3621                .roles
3622                .implementer_roster
3623                .iter()
3624                .position(|s| s.id == job.spec.id)
3625                .unwrap_or(0);
3626            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627            let mut fallback_attempt = 0usize;
3628            let mut prev: Option<FailClass> = None;
3629            while let Some(cur) = FailClass::of(&*out) {
3630                if !should_hand_over(prev.as_ref(), &cur) {
3631                    break;
3632                }
3633                let Some(next) =
3634                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635                        .cloned()
3636                else {
3637                    break;
3638                };
3639                tried.insert(next.id.clone());
3640                fallback_attempt += 1;
3641
3642                let coordination = self.coordinate(&job.cwd).await?;
3643                if let Ok(r) = git::rescue_commit(
3644                    &job.cwd,
3645                    &format!(
3646                        "magi: candidate {} (uncommitted work before {} fallback)",
3647                        seat.key,
3648                        if cur == FailClass::Quota {
3649                            "quota"
3650                        } else {
3651                            "handover"
3652                        }
3653                    ),
3654                )
3655                .await
3656                {
3657                    self.state.note_withheld("implement", &r.withheld);
3658                }
3659                drop(coordination);
3660
3661                record_handover(
3662                    &mut self.state,
3663                    "implement",
3664                    &seat.key,
3665                    &seat.agent,
3666                    &next.id,
3667                    &cur,
3668                    &fail_reason(&*out),
3669                );
3670                prev = Some(cur.clone());
3671
3672                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674                // Kept in sync on `sent` itself, not just the local retry: a
3675                // later helper (`resume_unconfirmed_commands`) reads `sent`
3676                // after this one returns and must see whichever agent is now
3677                // occupying the seat, not the one that just quota'd out —
3678                // otherwise it would judge session/continuation eligibility
3679                // by the wrong CLI and could resend a fallback's session id
3680                // to the agent that lost it the seat in the first place.
3681                job.spec = next.clone();
3682                let mut retry = job.clone();
3683                retry.seat = new_seat;
3684                retry.prompt = prompt::implement_in(
3685                    &instruction,
3686                    &job.cwd.to_string_lossy(),
3687                    &language,
3688                    &github_language,
3689                    brief.as_deref(),
3690                    &attachments,
3691                );
3692                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693                let cache = self.state.config.cache_dir();
3694                let ctx = WaveCtx {
3695                    carry_seats: false,
3696                    pauses: Some((&self.pause, &self.interrupt)),
3697                    run: run_id,
3698                    node: "implement",
3699                    prompts,
3700                    cache: cache.as_deref(),
3701                    round: None,
3702                };
3703                let (fallback_seat, fallback_out) = run_one(
3704                    retry,
3705                    Arc::clone(&self.sem),
3706                    &ctx,
3707                    &mut self.state,
3708                    fallback_attempt,
3709                )
3710                .await;
3711                *seat = fallback_seat;
3712                *out = fallback_out;
3713            }
3714        }
3715        Ok(())
3716    }
3717
3718    /// Ask an implement seat's own CLI to confirm what it started, once, when
3719    /// its reply reported a command whose completion status it never
3720    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3721    /// that does and does not mean.
3722    ///
3723    /// The completion contract this task asks for, extended to `implement`
3724    /// with the same signal `continue_fix_report` reads for the fixer,
3725    /// rather than a keyword search over the reply or a hard requirement on
3726    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3727    /// a candidate's CLI turn ended cleanly while a test run it had started
3728    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3729    /// named in it at all is untouched by this: `commands` is empty, so
3730    /// there is nothing to be unconfirmed.
3731    ///
3732    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3733    /// this is not about recovering edits that might already be on disk, it
3734    /// is about a result the seat itself never vouched for, which resuming
3735    /// asks for regardless of what the tree already holds. Bounded to one
3736    /// attempt for the same reason `resume_undelivered` is — this is the
3737    /// most expensive node in the graph — and a seat that still cannot
3738    /// confirm on that attempt is left as whatever its (possibly still
3739    /// unconfirmed) reply says; this does not invent a new "failed" reason
3740    /// for a candidate that otherwise produced a real, committed change.
3741    async fn resume_unconfirmed_commands(
3742        &mut self,
3743        results: &mut [(usize, SeatState, AgentOutcome)],
3744        sent: &[SeatJob],
3745        prompts: &Prompts,
3746        run_id: &str,
3747    ) {
3748        for (wi, seat, out) in results.iter_mut() {
3749            let AgentOutcome::Ok(o) = &*out else {
3750                continue;
3751            };
3752            if !has_unconfirmed_command(&o.commands) {
3753                continue;
3754            }
3755            let Some(job) = sent.get(*wi) else { continue };
3756            if !has_context(&job.spec, seat, job.sessions) {
3757                self.state.event(
3758                    "implement",
3759                    format!(
3760                        "{}: the reply named a command whose own CLI never confirmed the exit \
3761                         status of, but there is no session left to resume",
3762                        seat.key
3763                    ),
3764                );
3765                continue;
3766            }
3767            self.state.event(
3768                "implement",
3769                format!(
3770                    "{}: the reply named a command whose own CLI never confirmed the exit \
3771                     status of; resuming the conversation",
3772                    seat.key
3773                ),
3774            );
3775            let mut retry = job.clone();
3776            retry.seat = seat.clone();
3777            retry.prompt = prompt::resume_incomplete(
3778                "a command in your last reply had no confirmed exit status",
3779            );
3780            retry.timeout = retry_budget(job.timeout, true);
3781            retry.stem = format!("{}-confirm", job.stem);
3782            let cache = self.state.config.cache_dir();
3783            let ctx = WaveCtx {
3784                carry_seats: false,
3785                pauses: Some((&self.pause, &self.interrupt)),
3786                run: run_id,
3787                node: "implement",
3788                prompts,
3789                cache: cache.as_deref(),
3790                round: None,
3791            };
3792            let (resumed_seat, resumed) =
3793                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794            *seat = resumed_seat;
3795            *out = resumed;
3796        }
3797    }
3798
3799    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3800    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3801    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3802    /// that motivated this.
3803    ///
3804    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3805    /// own nudge loop already covers for judge/review/vote seats, and not a
3806    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3807    /// implement seats: here the CLI turn genuinely finished while the node's
3808    /// own work — the fixer's account of what it did — had not. Gated purely
3809    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3810    /// reply, never on any wording in it, so a fixer whose valid, first-try
3811    /// `FixReport` happens to mention having waited on a background test is
3812    /// never resumed — the `Ok(report)` branch at the call site returns
3813    /// before this is ever invoked.
3814    ///
3815    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3816    /// attempt ([`retry_budget`]), nothing attempted once the session is
3817    /// gone, and a quota hit ends the loop immediately rather than retrying a
3818    /// rate limit that fails the same way again.
3819    async fn continue_fix_report(
3820        &mut self,
3821        mut seat: SeatState,
3822        parse_err: String,
3823        job: &SeatJob,
3824        prompts: &Prompts,
3825        run_id: &str,
3826        round: usize,
3827    ) -> (
3828        SeatState,
3829        Option<FixReport>,
3830        Option<String>,
3831        ContinuationRecord,
3832    ) {
3833        let mut last_err = parse_err;
3834        let mut cumulative_wait_ms = 0u64;
3835        let mut attempts = 0usize;
3836        loop {
3837            if !has_context(&job.spec, &seat, job.sessions) {
3838                self.state.event(
3839                    "fix",
3840                    format!(
3841                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842                         session left to resume into"
3843                    ),
3844                );
3845                let outcome = if attempts == 0 {
3846                    ContinuationOutcome::NoSession
3847                } else {
3848                    ContinuationOutcome::Exhausted
3849                };
3850                return (
3851                    seat,
3852                    None,
3853                    Some(format!("unparsable fix report: {last_err}")),
3854                    ContinuationRecord {
3855                        attempts,
3856                        cumulative_wait_ms,
3857                        outcome,
3858                    },
3859                );
3860            }
3861            if attempts >= MAX_FIX_CONTINUATIONS {
3862                self.state.event(
3863                    "fix",
3864                    format!(
3865                        "round {round}: fixer's reply still had no adoption report after \
3866                         {attempts} continuation(s) ({last_err}); giving up"
3867                    ),
3868                );
3869                return (
3870                    seat,
3871                    None,
3872                    Some(format!(
3873                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3874                    )),
3875                    ContinuationRecord {
3876                        attempts,
3877                        cumulative_wait_ms,
3878                        outcome: ContinuationOutcome::Exhausted,
3879                    },
3880                );
3881            }
3882            attempts += 1;
3883            self.state.event(
3884                "fix",
3885                format!(
3886                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888                ),
3889            );
3890            let mut retry = job.clone();
3891            retry.seat = seat.clone();
3892            retry.prompt = prompt::resume_incomplete(&last_err);
3893            retry.timeout = retry_budget(job.timeout, true);
3894            retry.stem = format!("{}-continue{attempts}", job.stem);
3895            let cache = self.state.config.cache_dir();
3896            let ctx = WaveCtx {
3897                carry_seats: false,
3898                pauses: Some((&self.pause, &self.interrupt)),
3899                run: run_id,
3900                node: "fix",
3901                prompts,
3902                cache: cache.as_deref(),
3903                round: Some(round),
3904            };
3905            let (resumed_seat, resumed_out) = run_one(
3906                retry,
3907                Arc::clone(&self.sem),
3908                &ctx,
3909                &mut self.state,
3910                attempts,
3911            )
3912            .await;
3913            seat = resumed_seat;
3914            match resumed_out {
3915                AgentOutcome::Ok(o) => {
3916                    cumulative_wait_ms += o.duration_ms;
3917                    match verdict::extract_json::<FixReport>(&o.text) {
3918                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919                            self.state.event(
3920                                "fix",
3921                                format!(
3922                                    "round {round}: fixer's adoption report recovered after \
3923                                     {attempts} continuation(s)"
3924                                ),
3925                            );
3926                            return (
3927                                seat,
3928                                Some(report),
3929                                None,
3930                                ContinuationRecord {
3931                                    attempts,
3932                                    cumulative_wait_ms,
3933                                    outcome: ContinuationOutcome::Resumed,
3934                                },
3935                            );
3936                        }
3937                        // The report parsed, but this same reply's own
3938                        // CommandEvidence — the identical record `state.jobs`
3939                        // renders — names a command whose CLI never
3940                        // confirmed an exit status. Read together, that is
3941                        // not a resolved answer: keep nudging rather than
3942                        // accept a report standing next to a command the
3943                        // seat's own CLI cannot vouch for.
3944                        Ok(_) => {
3945                            last_err = "the reply parsed, but it reported a command whose own CLI \
3946                                 never confirmed an exit status"
3947                                .to_owned();
3948                        }
3949                        Err(e) => last_err = e.to_string(),
3950                    }
3951                }
3952                AgentOutcome::Quota(o) => {
3953                    cumulative_wait_ms += o.duration_ms;
3954                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956                    self.state.quota.push(loss);
3957                    self.state.event(
3958                        "fix",
3959                        format!(
3960                            "round {round}: continuation rate limited (quota); not retrying now"
3961                        ),
3962                    );
3963                    return (
3964                        seat,
3965                        None,
3966                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3967                        ContinuationRecord {
3968                            attempts,
3969                            cumulative_wait_ms,
3970                            outcome: ContinuationOutcome::QuotaLost,
3971                        },
3972                    );
3973                }
3974                AgentOutcome::Dropped(o) => {
3975                    cumulative_wait_ms += o.duration_ms;
3976                    let why = o
3977                        .dropped
3978                        .as_ref()
3979                        .map(|d| d.why.as_str())
3980                        .unwrap_or("the CLI ended the stream without delivering its answer");
3981                    last_err = format!("the CLI dropped the stream ({why})");
3982                }
3983                AgentOutcome::Failed(e) => last_err = e,
3984                // The continuation was never started; the report stays lost
3985                // and the caller records that, without any handover.
3986                AgentOutcome::ResourceBlocked(e) => {
3987                    return (
3988                        seat,
3989                        None,
3990                        Some(format!("the continuation was not started: {e}")),
3991                        ContinuationRecord {
3992                            attempts: attempts.saturating_sub(1),
3993                            cumulative_wait_ms,
3994                            outcome: ContinuationOutcome::NoSession,
3995                        },
3996                    );
3997                }
3998            }
3999        }
4000    }
4001
4002    fn after_implement(&mut self) -> Result<()> {
4003        if !self.state.candidate_preservation.is_empty() {
4004            bail!("candidate preservation incomplete; judging is unsafe");
4005        }
4006        // Scan every candidate patch once the set is complete.
4007        if self.state.leaks.is_empty() {
4008            let cfg = self.state.config.blind.clone();
4009            let mut leaks = Vec::new();
4010            for c in &self.state.candidates {
4011                let Some(patch) =
4012                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013                else {
4014                    continue;
4015                };
4016                leaks.extend(blind::scan(
4017                    &format!("candidate {} patch", c.label),
4018                    &patch,
4019                    &cfg.vendor_tokens,
4020                ));
4021            }
4022            if !leaks.is_empty() {
4023                let summary = leaks
4024                    .iter()
4025                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026                    .collect::<Vec<_>>()
4027                    .join(", ");
4028                match cfg.on_leak {
4029                    LeakPolicy::Fail => {
4030                        self.state.status = RunStatus::Failed;
4031                        self.state
4032                            .event("blind", format!("vendor text in a patch: {summary}"));
4033                        self.state.leaks = leaks;
4034                        self.state.save()?;
4035                        self.settle_questions();
4036                        bail!(
4037                            "blind.on_leak = \"fail\" and vendor text reached a \
4038                             judged patch: {summary}"
4039                        );
4040                    }
4041                    LeakPolicy::Redact => self.state.event(
4042                        "blind",
4043                        format!("redacting vendor text for judging: {summary}"),
4044                    ),
4045                    LeakPolicy::Warn => self.state.event(
4046                        "blind",
4047                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048                    ),
4049                }
4050                self.state.leaks = leaks;
4051            }
4052        }
4053
4054        if self.state.viable().is_empty() {
4055            if self.state.all_candidates_verified_noop() {
4056                // Every candidate agreed, with evidence the adoption guard
4057                // accepted, that nothing belongs in this worktree. That is
4058                // not the same fact as a candidate that simply failed to
4059                // write anything, and settling it as an ordinary `Failed`
4060                // (see `SCHEMA`'s doc for schema 10) is what let two of
4061                // task 391f's attempts burn a retry each re-discovering the
4062                // same already-landed fix. Terminal either way, so `judge`
4063                // must never run over an empty candidate set — unlike the
4064                // `Failed` branch below this returns `Ok`, not an error:
4065                // nothing here failed.
4066                self.state.status = RunStatus::VerifiedNoop;
4067                self.state.save()?;
4068                self.settle_questions();
4069                return Ok(());
4070            }
4071            self.state.status = RunStatus::Failed;
4072            self.state.save()?;
4073            self.settle_questions();
4074            bail!("no candidate produced a change; nothing to judge");
4075        }
4076        self.state.status = RunStatus::Judging;
4077        self.state.save()?;
4078        Ok(())
4079    }
4080
4081    // --------------------------------------------------------------- judge
4082
4083    async fn judge(&mut self) -> Result<()> {
4084        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4085        // agent files with `magi task add` name the run that paid for it. The
4086        // prompt overlay is cloned alongside it because the waves borrow it
4087        // while `self` is mutably borrowed by the node's own bookkeeping.
4088        let run_id = self.state.id.clone();
4089        let prompts = self.state.config.prompts.clone();
4090        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091            return Ok(());
4092        }
4093        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094        if viable.len() == 1 {
4095            // Recorded so this is a one-time event: `judgements` stays empty
4096            // either way, which without this flag is indistinguishable from
4097            // "not yet judged" on the next reentry — and status is left
4098            // untouched, so a later node's conclusion (e.g. `Blocked` after
4099            // the review budget ran out) survives a resume instead of being
4100            // clobbered back to `Judging` by this node running again.
4101            self.state.judge_skipped = true;
4102            self.state.event(
4103                "judge",
4104                format!(
4105                    "only candidate {} produced a change; judging skipped",
4106                    viable[0].label
4107                ),
4108            );
4109            self.state.save()?;
4110            return Ok(());
4111        }
4112        self.state.status = RunStatus::Judging;
4113
4114        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115        let language = self.state.config.graph.language.clone();
4116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117        let sessions = self.state.config.graph.sessions;
4118        let artifacts = agent::artifacts_dir(&self.state.dir());
4119        let root = self.state.worktree_root();
4120        let base_short = short(&self.state.base_commit);
4121
4122        let mut jobs = Vec::new();
4123        let mut orders = Vec::new();
4124        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128            let seat_key = format!("judge-{}", j + 1);
4129            let seat = self.seat(&seat_key, &spec.id);
4130            jobs.push(SeatJob {
4131                prompt: prompt::judge(
4132                    &self.state.instruction,
4133                    &views,
4134                    self.roles.judges.len(),
4135                    &base_short,
4136                    &language,
4137                ),
4138                spec,
4139                seat,
4140                cwd: root.join(format!("judge-{}", j + 1)),
4141                timeout,
4142                allow_write: false,
4143                sessions,
4144                artifacts: artifacts.clone(),
4145                stem: format!("judge-{}", j + 1),
4146                handover: None,
4147            });
4148        }
4149
4150        self.state.event(
4151            "judge",
4152            format!(
4153                "{} judges ranking {} candidates blind",
4154                jobs.len(),
4155                viable.len()
4156            ),
4157        );
4158        let labels_for_check = labels.clone();
4159        let mut quota_losses = Vec::new();
4160        let cache = self.state.config.cache_dir();
4161        let ctx = WaveCtx {
4162            carry_seats: false,
4163            pauses: Some((&self.pause, &self.interrupt)),
4164            run: &run_id,
4165            node: "judge",
4166            prompts: &prompts,
4167            cache: cache.as_deref(),
4168            round: None,
4169        };
4170        let results = ask_json_wave::<Ranking>(
4171            jobs,
4172            Arc::clone(&self.sem),
4173            self.state.config.graph.retries,
4174            &self.roles.judge_roster,
4175            &ctx,
4176            &mut quota_losses,
4177            &mut self.state,
4178            &move |r: &Ranking| r.validate(&labels_for_check),
4179        )
4180        .await;
4181        if self.state.resource_block.is_some() {
4182            return self.stop_for_resource();
4183        }
4184        self.state.quota.extend(quota_losses);
4185
4186        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187            let agent_id = seat.agent.clone();
4188            self.state.seats.insert(seat.key.clone(), seat);
4189            let mut record = Judgement {
4190                judge: j + 1,
4191                seat: format!("judge-{}", j + 1),
4192                agent: agent_id,
4193                ranking: Vec::new(),
4194                reasons: BTreeMap::new(),
4195                confidence: None,
4196                strengths: BTreeMap::new(),
4197                order: orders[j].clone(),
4198                failed: None,
4199                duration_ms: 0,
4200            };
4201            match res {
4202                Ok((ranking, out)) => {
4203                    record.ranking = ranking.normalized();
4204                    record.reasons = ranking.reasons;
4205                    record.strengths = ranking.strengths;
4206                    record.confidence = ranking.confidence;
4207                    record.duration_ms = out.duration_ms;
4208                    self.state.event(
4209                        "judge",
4210                        format!(
4211                            "judge {} ranked {}",
4212                            j + 1,
4213                            record.ranking.iter().collect::<String>()
4214                        ),
4215                    );
4216                }
4217                Err(e) => {
4218                    record.failed = Some(e.to_string());
4219                    self.state
4220                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221                }
4222            }
4223            self.state.judgements.push(record);
4224            self.state.save()?;
4225        }
4226        Ok(())
4227    }
4228
4229    // ---------------------------------------------------------- deliberate
4230
4231    async fn deliberate(&mut self) -> Result<()> {
4232        if self.hold_base_sync().await? {
4233            return Ok(());
4234        }
4235        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4236        // agent files with `magi task add` name the run that paid for it. The
4237        // prompt overlay is cloned alongside it because the waves borrow it
4238        // while `self` is mutably borrowed by the node's own bookkeeping.
4239        let run_id = self.state.id.clone();
4240        let prompts = self.state.config.prompts.clone();
4241        let rounds = self.state.config.graph.deliberate_rounds;
4242        // Resume unfinished rounds after an outage, but never reopen a tally.
4243        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244            return Ok(());
4245        }
4246        let tops: Vec<char> = self
4247            .state
4248            .judgements
4249            .iter()
4250            .filter_map(|j| j.ranking.first().copied())
4251            .collect();
4252        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254                self.state.event(
4255                    "deliberate",
4256                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4257                );
4258            }
4259            self.state.status = RunStatus::Voting;
4260            self.state.save()?;
4261            return Ok(());
4262        }
4263
4264        self.state.status = RunStatus::Deliberating;
4265        self.state.event(
4266            "deliberate",
4267            format!(
4268                "split: first choices were {} — opening {rounds} round(s)",
4269                tops.iter().collect::<String>()
4270            ),
4271        );
4272
4273        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274        let language = self.state.config.graph.language.clone();
4275        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276        let sessions = self.state.config.graph.sessions;
4277        let artifacts = agent::artifacts_dir(&self.state.dir());
4278        let root = self.state.worktree_root();
4279        let base_short = short(&self.state.base_commit);
4280
4281        // Judges argue in sequence so that a turn can answer the one before it;
4282        // that is the difference between deliberation and three parallel
4283        // monologues.
4284        for round in self.state.deliberation.len() + 1..=rounds {
4285            let mut turns: Vec<DeliberationTurn> = Vec::new();
4286            let mut round_losses = Vec::new();
4287            let mut asked = 0;
4288            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289                if self.state.judgements[j].failed.is_some() {
4290                    continue;
4291                }
4292                let seat_key = format!("judge-{}", j + 1);
4293                let spec = self.occupant(&seat_key, spec);
4294                let mut seat = self.seat(&seat_key, &spec.id);
4295                let transcript = self.transcript(&turns, j);
4296                let build = |context: Option<&str>| {
4297                    prompt::deliberate(
4298                        &self.state.instruction,
4299                        context,
4300                        &transcript,
4301                        round,
4302                        rounds,
4303                        &language,
4304                    )
4305                };
4306                let block = self.candidate_block(&viable, &base_short);
4307                let full = build(Some(&block));
4308                let text = if has_context(&spec, &seat, sessions) {
4309                    build(None)
4310                } else {
4311                    full.clone()
4312                };
4313                let job = SeatJob {
4314                    spec,
4315                    seat: seat.clone(),
4316                    prompt: text,
4317                    cwd: root.join(format!("judge-{}", j + 1)),
4318                    timeout,
4319                    allow_write: false,
4320                    sessions,
4321                    artifacts: artifacts.clone(),
4322                    stem: format!("delib-{round}-judge-{}", j + 1),
4323                    handover: Some(full),
4324                };
4325                let cache = self.state.config.cache_dir();
4326                let ctx = WaveCtx {
4327                    carry_seats: false,
4328                    pauses: Some((&self.pause, &self.interrupt)),
4329                    run: &run_id,
4330                    node: "deliberate",
4331                    prompts: &prompts,
4332                    cache: cache.as_deref(),
4333                    round: None,
4334                };
4335                // A turn is never nudged (`retries` 0): a failed seat is
4336                // handed to the next roster agent, which gets the full
4337                // context. An empty answer is a turn, not a failure.
4338                let mut losses = Vec::new();
4339                asked += 1;
4340                let mut results = ask_wave_with::<String>(
4341                    vec![job],
4342                    Arc::clone(&self.sem),
4343                    0,
4344                    &self.roles.judge_roster,
4345                    &ctx,
4346                    &mut losses,
4347                    &mut self.state,
4348                    &|text: &str| {
4349                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350                    },
4351                )
4352                .await;
4353                if self.state.resource_block.is_some() {
4354                    return self.stop_for_resource();
4355                }
4356                round_losses.extend(losses.iter().cloned());
4357                self.state.quota.extend(losses);
4358                let (updated, res, _) = results.pop().expect("one job in, one result out");
4359                seat = updated;
4360                let agent_id = seat.agent.clone();
4361                self.state.seats.insert(seat.key.clone(), seat);
4362                let body = match res {
4363                    Ok((body, _)) => body,
4364                    // Skip the seat; a CLI's raw error JSON is never read as
4365                    // this judge's position.
4366                    Err(e) => {
4367                        self.state
4368                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369                        continue;
4370                    }
4371                };
4372                let tentative = verdict::extract_json::<Position>(&body)
4373                    .ok()
4374                    .and_then(|p| p.tentative)
4375                    .and_then(|s| s.trim().chars().next())
4376                    .map(|c| c.to_ascii_uppercase());
4377                self.state.event(
4378                    "deliberate",
4379                    format!(
4380                        "round {round}: judge {} now favours {}",
4381                        j + 1,
4382                        tentative.map_or("—".to_owned(), |c| c.to_string())
4383                    ),
4384                );
4385                turns.push(DeliberationTurn {
4386                    judge: j + 1,
4387                    agent: agent_id,
4388                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4389                    tentative,
4390                });
4391            }
4392            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394                return self.stop_for_resource();
4395            }
4396            self.state
4397                .deliberation
4398                .push(DeliberationRound { round, turns });
4399            self.state.save()?;
4400        }
4401
4402        self.state.status = RunStatus::Voting;
4403        self.state.save()?;
4404        Ok(())
4405    }
4406
4407    // ---------------------------------------------------------------- vote
4408
4409    async fn vote(&mut self) -> Result<()> {
4410        if self.hold_base_sync().await? {
4411            return Ok(());
4412        }
4413        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4414        // agent files with `magi task add` name the run that paid for it. The
4415        // prompt overlay is cloned alongside it because the waves borrow it
4416        // while `self` is mutably borrowed by the node's own bookkeeping.
4417        let run_id = self.state.id.clone();
4418        let prompts = self.state.config.prompts.clone();
4419        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420            return Ok(());
4421        }
4422        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423        if viable.len() == 1 {
4424            return Ok(());
4425        }
4426        self.state.status = RunStatus::Voting;
4427
4428        let language = self.state.config.graph.language.clone();
4429        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430        let sessions = self.state.config.graph.sessions;
4431        let artifacts = agent::artifacts_dir(&self.state.dir());
4432        let root = self.state.worktree_root();
4433        let base_short = short(&self.state.base_commit);
4434        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436        let mut jobs = Vec::new();
4437        let mut seats_at = Vec::new();
4438        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439            if self
4440                .state
4441                .judgements
4442                .get(j)
4443                .is_some_and(|r| r.failed.is_some())
4444            {
4445                continue;
4446            }
4447            let seat_key = format!("judge-{}", j + 1);
4448            let spec = self.occupant(&seat_key, spec);
4449            let seat = self.seat(&seat_key, &spec.id);
4450            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451            let text = if has_context(&spec, &seat, sessions) {
4452                prompt::final_vote(&viable, &language)
4453            } else {
4454                full.clone()
4455            };
4456            jobs.push(SeatJob {
4457                spec,
4458                seat,
4459                prompt: text,
4460                cwd: root.join(format!("judge-{}", j + 1)),
4461                timeout,
4462                allow_write: false,
4463                sessions,
4464                artifacts: artifacts.clone(),
4465                stem: format!("vote-judge-{}", j + 1),
4466                handover: Some(full),
4467            });
4468            seats_at.push(j);
4469        }
4470
4471        self.state.event(
4472            "vote",
4473            format!(
4474                "collecting {} final votes one by one, privately",
4475                jobs.len()
4476            ),
4477        );
4478        let allowed = viable.clone();
4479        let mut quota_losses = Vec::new();
4480        let cache = self.state.config.cache_dir();
4481        let ctx = WaveCtx {
4482            carry_seats: false,
4483            pauses: Some((&self.pause, &self.interrupt)),
4484            run: &run_id,
4485            node: "vote",
4486            prompts: &prompts,
4487            cache: cache.as_deref(),
4488            round: None,
4489        };
4490        let results = ask_json_wave::<FinalVote>(
4491            jobs,
4492            Arc::clone(&self.sem),
4493            self.state.config.graph.retries,
4494            &self.roles.judge_roster,
4495            &ctx,
4496            &mut quota_losses,
4497            &mut self.state,
4498            &move |v: &FinalVote| match v.label() {
4499                Some(c) if allowed.contains(&c) => Ok(()),
4500                other => bail!("vote {other:?} is not one of {allowed:?}"),
4501            },
4502        )
4503        .await;
4504        if self.state.resource_block.is_some() {
4505            return self.stop_for_resource();
4506        }
4507        self.state.quota.extend(quota_losses);
4508
4509        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510            let agent_id = seat.agent.clone();
4511            self.state.seats.insert(seat.key.clone(), seat);
4512            let initial = self
4513                .state
4514                .judgements
4515                .get(j)
4516                .and_then(|r| r.ranking.first().copied());
4517            let mut record = VoteRecord {
4518                judge: j + 1,
4519                agent: agent_id,
4520                vote: None,
4521                reason: String::new(),
4522                changed: false,
4523            };
4524            match res {
4525                Ok((v, _)) => {
4526                    record.vote = v.label();
4527                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529                    self.state.event(
4530                        "vote",
4531                        format!(
4532                            "judge {} voted {}{}",
4533                            j + 1,
4534                            record.vote.unwrap_or('?'),
4535                            if record.changed { " (changed)" } else { "" }
4536                        ),
4537                    );
4538                }
4539                Err(e) => {
4540                    self.state
4541                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542                }
4543            }
4544            self.state.votes.push(record);
4545            self.state.save()?;
4546        }
4547        Ok(())
4548    }
4549
4550    // --------------------------------------------------------------- tally
4551
4552    fn tally(&mut self) -> Result<()> {
4553        if self.state.tally.is_some() {
4554            return Ok(());
4555        }
4556        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557        let tops: Vec<char> = self
4558            .state
4559            .judgements
4560            .iter()
4561            .filter_map(|j| j.ranking.first().copied())
4562            .collect();
4563        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565        // A judge whose private vote failed still counted once, in the initial
4566        // ranking; using it beats discarding a whole seat.
4567        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568        let mut cast: Vec<char> = Vec::new();
4569        for (i, j) in self.state.judgements.iter().enumerate() {
4570            let vote = self
4571                .state
4572                .votes
4573                .iter()
4574                .find(|v| v.judge == i + 1)
4575                .and_then(|v| v.vote)
4576                .or_else(|| j.ranking.first().copied());
4577            if let Some(v) = vote {
4578                *first_choice.entry(v).or_insert(0) += 1;
4579                cast.push(v);
4580            }
4581        }
4582
4583        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584        for j in &self.state.judgements {
4585            let n = j.ranking.len();
4586            for (pos, label) in j.ranking.iter().enumerate() {
4587                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588            }
4589        }
4590
4591        let best = first_choice.values().copied().max().unwrap_or(0);
4592        let mut leaders: Vec<char> = first_choice
4593            .iter()
4594            .filter(|(_, v)| **v == best)
4595            .map(|(k, _)| *k)
4596            .collect();
4597        let mut tie_break = None;
4598        if leaders.len() > 1 {
4599            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600            let borda_leaders: Vec<char> = leaders
4601                .iter()
4602                .copied()
4603                .filter(|l| borda[l] == top_borda)
4604                .collect();
4605            tie_break = Some(if borda_leaders.len() == 1 {
4606                format!(
4607                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608                    leaders.len()
4609                )
4610            } else {
4611                format!(
4612                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4613                    leaders.len()
4614                )
4615            });
4616            leaders = borda_leaders;
4617            leaders.sort_unstable();
4618        }
4619        let winner = *leaders
4620            .first()
4621            .or(viable.first())
4622            .context("no candidate to declare a winner from")?;
4623
4624        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626        let deliberated = !self.state.deliberation.is_empty();
4627
4628        // Whose verdict is this? A rate-limited seat is absent even if it
4629        // ranked before the limit hit, so presence is measured against the
4630        // recorded losses, not just "did a ranking ever appear".
4631        let quota_seats: std::collections::BTreeSet<&str> =
4632            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633        let mut present = 0usize;
4634        for (i, j) in self.state.judgements.iter().enumerate() {
4635            if quota_seats.contains(j.seat.as_str()) {
4636                continue;
4637            }
4638            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639            let voted = self
4640                .state
4641                .votes
4642                .iter()
4643                .any(|v| v.judge == i + 1 && v.vote.is_some());
4644            if ranked || voted {
4645                present += 1;
4646            }
4647        }
4648        // Strict majority of the configured panel. A bare majority is real
4649        // signal we can act on, while a minority verdict must never stand in
4650        // for a healthy one. A one-candidate run needs no panel at all, and
4651        // `judges` stays `0` rather than the roster size a panel that never
4652        // sat would otherwise be credited with.
4653        let needs_quorum = viable.len() > 1;
4654        let judges_total = if needs_quorum {
4655            self.roles.judges.len()
4656        } else {
4657            0
4658        };
4659        let quorum = if needs_quorum {
4660            judges_total / 2 + 1
4661        } else {
4662            0
4663        };
4664        let met_quorum = !needs_quorum || present >= quorum;
4665        let uncontested = (!needs_quorum).then(|| {
4666            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667        });
4668
4669        self.state.event(
4670            "tally",
4671            match &uncontested {
4672                Some(reason) => format!("winner {winner} — {reason}"),
4673                None => format!(
4674                    "winner {winner} — votes {} | initial {} | {} changed | \
4675                     {present}/{judges_total} judges{}",
4676                    first_choice
4677                        .iter()
4678                        .map(|(k, v)| format!("{k}:{v}"))
4679                        .collect::<Vec<_>>()
4680                        .join(" "),
4681                    if unanimous_initial {
4682                        "unanimous"
4683                    } else {
4684                        "split"
4685                    },
4686                    changed_votes,
4687                    if met_quorum {
4688                        String::new()
4689                    } else {
4690                        format!(" — below quorum ({quorum} required)")
4691                    },
4692                ),
4693            },
4694        );
4695        if !met_quorum {
4696            self.state.event(
4697                "stall",
4698                format!(
4699                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700                     the run stops here, resumable"
4701                ),
4702            );
4703        }
4704        self.state.tally = Some(Tally {
4705            first_choice,
4706            borda,
4707            winner,
4708            rankings: tops.len(),
4709            unanimous_initial,
4710            deliberated,
4711            changed_votes,
4712            unanimous_final,
4713            tie_break,
4714            judges: judges_total,
4715            present,
4716            quorum,
4717            met_quorum,
4718            uncontested,
4719        });
4720        self.state.status = if met_quorum {
4721            RunStatus::Reviewing
4722        } else {
4723            RunStatus::Stalled
4724        };
4725        self.state.save()?;
4726        Ok(())
4727    }
4728
4729    // ------------------------------------------------------------- recover
4730
4731    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4732    /// resumed toward completion once the transient cause clears.
4733    ///
4734    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4735    /// it toward the quorum, which is exactly the set of seats whose absence
4736    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4737    /// not depend on which node happened to hit the limit), or an ordinary
4738    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4739    /// seat is never disturbed.
4740    ///
4741    /// A seat that now answers with a usable ranking is "recovered": its
4742    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4743    /// `tally` counts it present again), and its vote re-collected. A seat that
4744    /// still fails keeps its loss and stays absent.
4745    ///
4746    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4747    /// to review/gate/merge), `false` when it is still below quorum (the run
4748    /// stays `Stalled`, still resumable for a later retry).
4749    #[allow(clippy::too_many_lines)]
4750    async fn recover_stall(&mut self) -> Result<bool> {
4751        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4752        // agent files with `magi task add` name the run that paid for it. The
4753        // prompt overlay is cloned alongside it because the waves borrow it
4754        // while `self` is mutably borrowed by the node's own bookkeeping.
4755        let run_id = self.state.id.clone();
4756        let prompts = self.state.config.prompts.clone();
4757        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4758        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4759        // is what keeps a non-quota collapse — or a quota loss recorded at the
4760        // deliberate node — from being a permanent dead-end on `--resume`.
4761        let quota_seats: BTreeSet<&str> =
4762            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763        let absent: Vec<String> = self
4764            .state
4765            .judgements
4766            .iter()
4767            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768            .map(|j| j.seat.clone())
4769            .collect();
4770        if absent.is_empty() {
4771            return Ok(false);
4772        }
4773        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774        if viable.len() <= 1 {
4775            return Ok(false);
4776        }
4777        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778        let language = self.state.config.graph.language.clone();
4779        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780        let sessions = self.state.config.graph.sessions;
4781        let artifacts = agent::artifacts_dir(&self.state.dir());
4782        let root = self.state.worktree_root();
4783        let base_short = short(&self.state.base_commit);
4784        let candidates: Vec<Candidate> = viable.clone();
4785
4786        // Map each absent seat key to its 0-based position in `roles.judges`.
4787        let mut positions: Vec<usize> = absent
4788            .iter()
4789            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790            .collect();
4791        if positions.is_empty() {
4792            return Ok(false);
4793        }
4794        positions.sort_unstable();
4795        positions.dedup();
4796
4797        // Re-rank the lost seats, one blind prompt each.
4798        let mut judge_jobs = Vec::new();
4799        for &j in &positions {
4800            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802            let seat_key = format!("judge-{}", j + 1);
4803            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804            let seat = self.seat(&seat_key, &spec.id);
4805            judge_jobs.push(SeatJob {
4806                spec,
4807                seat,
4808                prompt: prompt::judge(
4809                    &self.state.instruction,
4810                    &views,
4811                    self.roles.judges.len(),
4812                    &base_short,
4813                    &language,
4814                ),
4815                cwd: root.join(seat_key),
4816                timeout,
4817                allow_write: false,
4818                sessions,
4819                artifacts: artifacts.clone(),
4820                stem: format!("judge-{}-recover", j + 1),
4821                handover: None,
4822            });
4823        }
4824
4825        let labels_for_check = labels.clone();
4826        let mut judge_losses = Vec::new();
4827        let retries = self.state.config.graph.retries;
4828        let cache = self.state.config.cache_dir();
4829        let ctx = WaveCtx {
4830            carry_seats: false,
4831            pauses: Some((&self.pause, &self.interrupt)),
4832            run: &run_id,
4833            node: "judge",
4834            prompts: &prompts,
4835            cache: cache.as_deref(),
4836            round: None,
4837        };
4838        let results = ask_json_wave::<Ranking>(
4839            judge_jobs,
4840            Arc::clone(&self.sem),
4841            retries,
4842            &self.roles.judge_roster,
4843            &ctx,
4844            &mut judge_losses,
4845            &mut self.state,
4846            &move |r: &Ranking| r.validate(&labels_for_check),
4847        )
4848        .await;
4849        if self.state.resource_block.is_some() {
4850            self.stop_for_resource()?;
4851            return Ok(false);
4852        }
4853
4854        // Refresh the judgement of every seat that ranked again.
4855        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857            let agent_id = seat.agent.clone();
4858            self.state.seats.insert(seat.key.clone(), seat);
4859            let record = &mut self.state.judgements[j];
4860            match res {
4861                Ok((ranking, out)) => {
4862                    record.agent = agent_id;
4863                    record.ranking = ranking.normalized();
4864                    record.reasons = ranking.reasons;
4865                    record.strengths = ranking.strengths;
4866                    record.confidence = ranking.confidence;
4867                    record.failed = None;
4868                    record.duration_ms = out.duration_ms;
4869                    recovered.insert(j);
4870                    self.state.event(
4871                        "recover",
4872                        format!("judge {} ranked again after the limit", j + 1),
4873                    );
4874                }
4875                Err(e) => {
4876                    self.state
4877                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878                }
4879            }
4880        }
4881
4882        // Re-ask the votes of the seats that recovered a ranking.
4883        let mut vote_jobs = Vec::new();
4884        let mut vote_pos: Vec<usize> = Vec::new();
4885        for &j in &recovered {
4886            let seat_key = format!("judge-{}", j + 1);
4887            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888            let seat = self.seat(&seat_key, &spec.id);
4889            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890            let text = if has_context(&spec, &seat, sessions) {
4891                prompt::final_vote(&labels, &language)
4892            } else {
4893                full.clone()
4894            };
4895            vote_jobs.push(SeatJob {
4896                spec,
4897                seat,
4898                prompt: text,
4899                cwd: root.join(seat_key),
4900                timeout,
4901                allow_write: false,
4902                sessions,
4903                artifacts: artifacts.clone(),
4904                stem: format!("vote-judge-{}-recover", j + 1),
4905                handover: Some(full),
4906            });
4907            vote_pos.push(j);
4908        }
4909        let allowed = labels.clone();
4910        let mut vote_losses = Vec::new();
4911        let vote_retries = self.state.config.graph.retries;
4912        let vote_cache = self.state.config.cache_dir();
4913        let ctx = WaveCtx {
4914            carry_seats: false,
4915            pauses: Some((&self.pause, &self.interrupt)),
4916            run: &run_id,
4917            node: "vote",
4918            prompts: &prompts,
4919            cache: vote_cache.as_deref(),
4920            round: None,
4921        };
4922        let votes = ask_json_wave::<FinalVote>(
4923            vote_jobs,
4924            Arc::clone(&self.sem),
4925            vote_retries,
4926            &self.roles.judge_roster,
4927            &ctx,
4928            &mut vote_losses,
4929            &mut self.state,
4930            &move |v: &FinalVote| match v.label() {
4931                Some(c) if allowed.contains(&c) => Ok(()),
4932                other => bail!("vote {other:?} is not one of {allowed:?}"),
4933            },
4934        )
4935        .await;
4936        if self.state.resource_block.is_some() {
4937            self.stop_for_resource()?;
4938            return Ok(false);
4939        }
4940        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941            let agent_id = seat.agent.clone();
4942            self.state.seats.insert(seat.key.clone(), seat);
4943            match res {
4944                Ok((v, _)) => {
4945                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946                        rec.vote = v.label();
4947                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948                    } else {
4949                        self.state.votes.push(VoteRecord {
4950                            judge: j + 1,
4951                            agent: agent_id,
4952                            vote: v.label(),
4953                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954                            changed: false,
4955                        });
4956                    }
4957                    self.state.event(
4958                        "recover",
4959                        format!("judge {} voted again after the limit", j + 1),
4960                    );
4961                }
4962                Err(e) => {
4963                    self.state
4964                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965                }
4966            }
4967        }
4968
4969        // A seat that ranked again is present even if its re-vote failed —
4970        // `tally` falls back to the initial ranking's first choice — so clear
4971        // its quota loss. Seats that still fail keep theirs and stay absent.
4972        let recovered_keys: BTreeSet<String> = recovered
4973            .iter()
4974            .map(|&j| format!("judge-{}", j + 1))
4975            .collect();
4976        self.state
4977            .quota
4978            .retain(|q| !recovered_keys.contains(&q.seat));
4979        // A seat that hit the limit again is a fresh loss, not the old one:
4980        // replace the stale entry so the history stays one-per-seat and the
4981        // daemon can tell this attempt's loss from a previous session's.
4982        for loss in judge_losses.into_iter().chain(vote_losses) {
4983            if recovered_keys.contains(&loss.seat) {
4984                continue;
4985            }
4986            self.state.quota.retain(|q| q.seat != loss.seat);
4987            self.state.quota.push(loss);
4988        }
4989
4990        // Recompute the verdict from the refreshed panel.
4991        self.state.tally = None;
4992        self.tally()?;
4993        Ok(self
4994            .state
4995            .tally
4996            .as_ref()
4997            .map(|t| t.met_quorum)
4998            .unwrap_or(false))
4999    }
5000
5001    // ---------------------------------------------------------------- graft
5002
5003    /// Port the strengths the judges attributed to the losing candidates onto
5004    /// the winner, once, between the tally and the review loop.
5005    ///
5006    /// Best-effort by construction: it returns nothing and fails nothing. A
5007    /// skip, a fixer that failed or hit quota, or a round that changed nothing
5008    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
5009    /// the run carries on as if the phase were off. See [`graft`] for the
5010    /// decisions and `AGENTS.md` for the invariants.
5011    ///
5012    /// The one exception is a rollback that could not be confirmed: the
5013    /// rejected commit may still be the winner's tip, so the run is blocked
5014    /// (resumable; the record stays `Running`, which makes a resume retry the
5015    /// restore) instead of reviewing, gating and merging it.
5016    async fn graft(&mut self) -> Result<()> {
5017        let Err(e) = self.graft_inner().await else {
5018            return Ok(());
5019        };
5020        if e.is::<crate::repo_lock::ResourceBlocked>() {
5021            return Err(e);
5022        }
5023        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024            self.state.status = RunStatus::Blocked;
5025            self.state.event("graft", format!("blocked: {e:#}"));
5026            self.state.save()?;
5027            return Err(e);
5028        }
5029        // An error after the `Running` record was saved may have left the
5030        // fixer's work in the tree: the record must not stay unsettled
5031        // (a later resume would reset past the review fixes), so confirm the
5032        // restore now, or block.
5033        if let Some(rec) = self.state.graft.clone()
5034            && !rec.outcome.settled()
5035        {
5036            if let Some(winner) = self.state.winner().cloned() {
5037                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038                    if re.is::<crate::repo_lock::ResourceBlocked>() {
5039                        return Err(re);
5040                    }
5041                    self.state.status = RunStatus::Blocked;
5042                    self.state.event("graft", format!("blocked: {re:#}"));
5043                    self.state.save()?;
5044                    return Err(re);
5045                }
5046            }
5047            if let Some(g) = self.state.graft.as_mut() {
5048                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049            }
5050        }
5051        self.state.event("graft", format!("skipped: {e:#}"));
5052        if let Err(e) = self.state.save() {
5053            tracing::warn!("save after graft: {e:#}");
5054        }
5055        Ok(())
5056    }
5057
5058    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
5059    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
5060    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061        let _coordination = self.coordinate(wt).await?;
5062        crate::recovery::restore_repair(wt, pre)
5063            .await
5064            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065    }
5066
5067    /// A skip is a run event, recorded once however often the node is re-entered.
5068    fn graft_skipped(&mut self, why: &str) {
5069        let message = format!("skipped: {why}");
5070        if !self
5071            .state
5072            .events
5073            .iter()
5074            .any(|e| e.node == "graft" && e.message == message)
5075        {
5076            self.state.event("graft", message);
5077        }
5078    }
5079
5080    async fn graft_inner(&mut self) -> Result<()> {
5081        // A round that was started and never finished (crash, kill): the
5082        // fixer may have left anything in the tree, so put the winner back.
5083        if let Some(rec) = self.state.graft.clone()
5084            && !rec.outcome.settled()
5085        {
5086            // Reviews exist only if the graft ended without its record being
5087            // settled (a swallowed error): the tree has moved on since, and
5088            // a reset to `pre` would discard the review fixes.
5089            if self.state.reviews.is_empty()
5090                && let Some(winner) = self.state.winner().cloned()
5091            {
5092                self.graft_restore(&winner.worktree, &rec.pre).await?;
5093            }
5094            if rec.outcome == GraftOutcome::WaitingForRestore {
5095                self.state.graft = None;
5096                self.state.save()?;
5097            } else {
5098                if let Some(g) = self.state.graft.as_mut() {
5099                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100                }
5101                self.state.event(
5102                    "graft",
5103                    "rolled back: the previous round was interrupted; the winner is untouched",
5104                );
5105                self.state.save()?;
5106                return Ok(());
5107            }
5108        }
5109        let plan = match graft::plan(&self.state) {
5110            Ok(plan) => plan,
5111            Err(skip) => {
5112                if skip != graft::Skip::AlreadyDone {
5113                    self.graft_skipped(skip.reason());
5114                    self.state.save()?;
5115                }
5116                return Ok(());
5117            }
5118        };
5119        let Some(winner) = self.state.winner().cloned() else {
5120            return Ok(());
5121        };
5122        let wt = winner.worktree.clone();
5123        if !git::is_clean(&wt).await? {
5124            self.graft_skipped("the winner's worktree has uncommitted changes");
5125            self.state.save()?;
5126            return Ok(());
5127        }
5128        let pre = git::rev_parse(&wt, "HEAD").await?;
5129        // Saved before the fixer is called: a stop from here on is repaired
5130        // by the branch above on the next entry.
5131        self.state.graft = Some(GraftRecord {
5132            pre: pre.clone(),
5133            post: None,
5134            lines: None,
5135            limit: None,
5136            agent: None,
5137            outcome: GraftOutcome::Running,
5138            requested: graft::requested(&plan),
5139            adoption: None,
5140            adoption_error: None,
5141        });
5142        self.state.event(
5143            "graft",
5144            format!(
5145                "porting strengths of candidate(s) {} onto {}",
5146                plan.strengths
5147                    .iter()
5148                    .map(|(l, _, _)| l.to_string())
5149                    .collect::<Vec<_>>()
5150                    .join(", "),
5151                plan.winner
5152            ),
5153        );
5154        self.state.save()?;
5155
5156        let fix_prompt = prompt::graft(
5157            &self.state.instruction,
5158            plan.winner,
5159            &plan.strengths,
5160            &self.state.config.graph.language,
5161        );
5162        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163        let sessions = self.state.config.graph.sessions;
5164        let artifacts = agent::artifacts_dir(&self.state.dir());
5165        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166        let (_, seat, out) = self
5167            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168                SeatJob {
5169                    prompt: fix_prompt.clone(),
5170                    spec,
5171                    seat,
5172                    cwd: wt.clone(),
5173                    timeout,
5174                    allow_write: true,
5175                    sessions,
5176                    artifacts: artifacts.clone(),
5177                    stem: "graft".to_owned(),
5178                    handover: None,
5179                }
5180            })
5181            .await;
5182        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183            // Preserve the need to restore earlier fallback edits without
5184            // confusing an unstarted seat with an interrupted invocation.
5185            self.state.graft.as_mut().expect("graft record").outcome =
5186                GraftOutcome::WaitingForRestore;
5187            return self.stop_for_resource();
5188        }
5189        let mut why: Option<String> = None;
5190        match &out {
5191            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192                why = Some("the fixer produced nothing usable".to_owned());
5193            }
5194            AgentOutcome::Ok(_) => {}
5195            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196            AgentOutcome::Quota(o) => {
5197                self.state.quota.push(QuotaLoss::from_report(
5198                    seat.key.clone(),
5199                    "graft",
5200                    o.quota.as_ref(),
5201                ));
5202                why = Some("rate limited (quota)".to_owned());
5203            }
5204            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205                why = Some(format!(
5206                    "the fixer failed: {}",
5207                    e.lines()
5208                        .next()
5209                        .unwrap_or("")
5210                        .chars()
5211                        .take(160)
5212                        .collect::<String>()
5213                ));
5214            }
5215        }
5216        let agent_id = seat.agent.clone();
5217        self.state.seats.insert(seat.key.clone(), seat);
5218
5219        let mut post = None;
5220        let mut lines = None;
5221        let _coordination = self.coordinate(&wt).await?;
5222        if why.is_none() {
5223            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224            if let Ok(r) = &rescued {
5225                self.state.note_withheld("graft", &r.withheld);
5226            }
5227            let head = git::rev_parse(&wt, "HEAD").await?;
5228            // What is measured must be what gets reviewed and gated: a failed
5229            // rescue, or anything still uncommitted (withheld lockfiles
5230            // aside, which are never part of the tree), rejects the graft.
5231            let dirty = match &rescued {
5232                Ok(r) => git::status_porcelain(&wt)
5233                    .await
5234                    .map(|st| {
5235                        st.lines().any(|l| {
5236                            !r.withheld
5237                                .iter()
5238                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239                        })
5240                    })
5241                    .unwrap_or(true),
5242                Err(_) => true,
5243            };
5244            if dirty {
5245                why = Some("the fixer left changes that could not be committed".to_owned());
5246            } else if head == pre {
5247                why = Some("the fixer changed nothing".to_owned());
5248            } else {
5249                lines = git::changed_lines(&wt, &pre, &head).await;
5250                post = Some(head);
5251            }
5252        }
5253        let (adoption, adoption_error) = match &out {
5254            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255            _ => (None, None),
5256        };
5257        let requested = graft::requested(&plan);
5258        let outcome = match why {
5259            None => {
5260                self.state.event(
5261                    "graft",
5262                    format!(
5263                        "applied: {}",
5264                        lines.map_or_else(
5265                            || "changed lines unavailable".to_owned(),
5266                            |n| format!("{n} changed lines")
5267                        )
5268                    ),
5269                );
5270                GraftOutcome::Applied
5271            }
5272            Some(why) => {
5273                // Not settled until the restore is confirmed: the record
5274                // stays `Running`, so a resume retries it.
5275                crate::recovery::restore_repair(&wt, &pre)
5276                    .await
5277                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278                self.state.event(
5279                    "graft",
5280                    format!("rolled back: {why}; the winner is untouched"),
5281                );
5282                GraftOutcome::RolledBack(why)
5283            }
5284        };
5285        self.state.graft = Some(GraftRecord {
5286            pre,
5287            post,
5288            lines,
5289            limit: None,
5290            agent: Some(agent_id),
5291            outcome,
5292            requested,
5293            adoption,
5294            adoption_error,
5295        });
5296        self.state.save()?;
5297        Ok(())
5298    }
5299
5300    // ----------------------------------------------------------------- fold
5301
5302    async fn fold_losers(&mut self) -> Result<()> {
5303        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304            return Ok(());
5305        };
5306        if !self.state.candidate_preservation.is_empty() {
5307            bail!("candidate preservation incomplete; refusing cleanup");
5308        }
5309        let repo = self.state.repo.clone();
5310        let _coordination = self.coordinate(&repo).await?;
5311        let mut folded = Vec::new();
5312        for i in 0..self.state.candidates.len() {
5313            let c = &self.state.candidates[i];
5314            if c.label == winner || c.folded {
5315                continue;
5316            }
5317            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318            remove_preserved_worktree(&repo, &wt).await?;
5319            if git::branch_exists(&repo, &branch).await?
5320                && !git::branch_delete(&repo, &branch).await?
5321            {
5322                bail!("candidate branch removal refused");
5323            }
5324            self.state.candidates[i].folded = true;
5325            self.state.save()?;
5326            folded.push(label.to_string());
5327        }
5328        // The judges are finished; their checkouts are pure cost from here.
5329        let root = self.state.worktree_root();
5330        for j in 1..=self.roles.judges.len() {
5331            let wt = root.join(format!("judge-{j}"));
5332            if wt.exists() {
5333                remove_read_seat_worktree(&mut self.state, &wt).await?;
5334                self.state.save()?;
5335            }
5336        }
5337        // The design-deliberation stage is finished by the time a tally
5338        // exists — same reasoning as the judges above.
5339        if self.state.config.graph.advise {
5340            for k in 1..=self.state.config.graph.advisors {
5341                let wt = root.join(format!("advisor-{k}"));
5342                if wt.exists() {
5343                    remove_read_seat_worktree(&mut self.state, &wt).await?;
5344                    self.state.save()?;
5345                }
5346            }
5347        }
5348        if !folded.is_empty() {
5349            self.state
5350                .event("fold", format!("folded candidates {}", folded.join(", ")));
5351            self.state.save()?;
5352        }
5353        Ok(())
5354    }
5355
5356    // ------------------------------------------------------------ base sync
5357
5358    async fn hold_base_sync(&mut self) -> Result<bool> {
5359        self.hold_base_sync_as(None).await
5360    }
5361
5362    async fn hold_base_sync_kind(
5363        &mut self,
5364        kind: crate::base_sync_recovery::StopKind,
5365    ) -> Result<()> {
5366        self.hold_base_sync_as(Some(kind)).await?;
5367        Ok(())
5368    }
5369
5370    async fn hold_base_sync_as(
5371        &mut self,
5372        kind: Option<crate::base_sync_recovery::StopKind>,
5373    ) -> Result<bool> {
5374        if !self.state.base_sync_blocked() {
5375            self.state.base_sync_recovery.stop = None;
5376            return Ok(false);
5377        }
5378        let head = match self.state.winner() {
5379            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5380                .await
5381                .unwrap_or_default(),
5382            None => String::new(),
5383        };
5384        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5385            && !authorization.consumed
5386            && authorization.stop.head != head
5387        {
5388            authorization.refuse(format!(
5389                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5390                authorization.stop.head
5391            ));
5392        }
5393        self.state.record_base_sync_stop(&head);
5394        if let Some(kind) = kind {
5395            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5396            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5397                stop.kind = kind;
5398            }
5399        }
5400        self.state.save()?;
5401        Ok(true)
5402    }
5403
5404    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5405    /// anything verifies it.
5406    ///
5407    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5408    /// read whatever is checked out in the winner's worktree. Left alone that
5409    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5410    /// it when the run *branched* - and a run takes long enough that the base
5411    /// has usually moved by the time it gets here. A gate that ran there
5412    /// answers "green on the commit this run started from", not "green on
5413    /// what is about to land", and the difference showed up three times in
5414    /// one day as a green run whose merge would have reverted a file another
5415    /// pull request had already landed.
5416    ///
5417    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5418    /// `land::Step::Rebase` calls, rather than a second implementation of the
5419    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5420    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5421    /// doc warns about ("two answers to a question nobody notices until a
5422    /// diff is wrong").
5423    ///
5424    /// Foreign-checkout reviews instead merge the base into the owned shadow
5425    /// worktree, preserving the ancestry required for fast-forward delivery.
5426    /// A conflicting merge is aborted and held for operator resolution.
5427    ///
5428    /// A conflict is not the end of the road: the standing rebase is handed
5429    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5430    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5431    /// land). Once it finishes, review and the gate run as usual on the
5432    /// rebased tree, which is where a breakage the new base caused is caught
5433    /// by the ordinary gate-fix round. magi resolves nothing itself.
5434    ///
5435    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5436    /// the current durable recovery interval, bounds how often the base is
5437    /// synchronized during active recovery (lifetime attempts remain evidence).
5438    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5439    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5440    /// what was tried (rounds spent, paths still conflicted) and the branch
5441    /// and worktree stay exactly as they were - untouched, for a person to
5442    /// look at - which is also what makes re-entering this function
5443    /// afterwards a no-op instead of a second attempt at the same wall. A
5444    /// push failure ends the same way.
5445    async fn sync_to_base(&mut self) -> Result<()> {
5446        if self.state.status == RunStatus::AlreadyInBase {
5447            return Ok(());
5448        }
5449        self.hold_base_sync().await?;
5450        let conflicted = self
5451            .state
5452            .base_sync
5453            .as_ref()
5454            .is_some_and(|s| s.conflict.is_some());
5455        let Some(winner) = self.state.winner().cloned() else {
5456            return Ok(());
5457        };
5458
5459        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5460
5461        let repo = self.state.repo.clone();
5462        let remote = self.state.config.merge.remote.clone();
5463        let base_branch = self.state.base_branch.clone();
5464        let tracking = format!("{remote}/{base_branch}");
5465
5466        let fetched_base =
5467            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5468        // No network, or the remote never had this branch: the run already
5469        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5470        // otherwise), and one that got this far is not blocked by a fetch
5471        // that fails now. It just has no newer tip to compare against.
5472        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5473            return Ok(());
5474        };
5475
5476        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5477        self.state.ensure_base_episode();
5478        let mut pending = self
5479            .state
5480            .base_sync_recovery
5481            .operations
5482            .last()
5483            .filter(|o| !o.finished)
5484            .cloned();
5485        let scratch = self.state.dir().join("base-sync");
5486        if git::rebase_in_progress(&scratch).await {
5487            // Older resource stops predate operation reservations. Import the
5488            // standing Git operation, never retarget it to today's main.
5489            if pending.is_none()
5490                && self.state.base_sync_recovery.operations.is_empty()
5491                && !conflicted
5492                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5493                    .await
5494                    .ok()
5495                    .as_deref()
5496                    == Some(head.as_str())
5497                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5498                && git::is_clean(&winner.worktree).await?
5499            {
5500                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5501                if self.state.base_sync.is_none() {
5502                    self.state.base_sync = Some(BaseSync {
5503                        tip: saved_tip.clone(),
5504                        behind: 0,
5505                        attempts: 0,
5506                        conflict: None,
5507                        already_in: None,
5508                    });
5509                }
5510                self.state.reserve_base_operation(&head, &saved_tip);
5511                self.state.event(
5512                    "base-recovery",
5513                    "imported standing legacy rebase as a spent synchronization reservation",
5514                );
5515                self.state.save()?;
5516                pending = self.state.base_sync_recovery.operations.last().cloned();
5517            }
5518            let expected = pending.as_ref().map(|op| op.tip.as_str());
5519            if expected.is_none()
5520                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5521            {
5522                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5523                if self.state.base_sync.is_none() {
5524                    self.state.base_sync = Some(BaseSync {
5525                        tip: tip.clone(),
5526                        behind: 0,
5527                        attempts: 0,
5528                        conflict: None,
5529                        already_in: None,
5530                    });
5531                }
5532                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5533                self.state.event("land", why);
5534                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535                    .await?;
5536                return Ok(());
5537            }
5538        }
5539        if let Some(op) = &pending {
5540            // A replay may add the pinned base's commits, never foreign work.
5541            let mut replay_proven =
5542                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5543            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5544                let wait = repository_wait(&self.state, None);
5545                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5546                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5547                    Ok(true) => {
5548                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5549                        self.state.save()?;
5550                        git::worktree_remove(&repo, &scratch).await.ok();
5551                        replay_proven = true;
5552                    }
5553                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5554                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5555                }
5556            }
5557            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5558                if self.state.foreign_checkout.is_some() {
5559                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5560                        .await
5561                        .ok()
5562                        .as_deref()
5563                        == Some(op.head.as_str())
5564                        && git::rev_parse(&repo, &format!("{head}^2"))
5565                            .await
5566                            .ok()
5567                            .as_deref()
5568                            == Some(op.tip.as_str());
5569                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5570                    .await
5571                    .is_ok_and(|m| m.is_empty())
5572                    && let Ok(added) =
5573                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5574                {
5575                    replay_proven = true;
5576                    for commit in added {
5577                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5578                            replay_proven = false;
5579                            break;
5580                        }
5581                    }
5582                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5583                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5584                }
5585            }
5586            // Never retarget an operation across a crash, even if main moved.
5587            // sync_to_head resets and cleans with -x, so even ignored untracked
5588            // files must stop recovery independently of the old tracked tree.
5589            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590                || (head != op.head
5591                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5592                        .await?
5593                        .is_empty())
5594                || (!git::is_clean(&winner.worktree).await?
5595                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5596                        .await
5597                        .is_ok_and(|o| o.ok()))
5598                || !replay_proven
5599            {
5600                let why = format!(
5601                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5602                    op.id
5603                );
5604                let sync = self
5605                    .state
5606                    .base_sync
5607                    .as_mut()
5608                    .context("reserved synchronization has no lifetime accounting")?;
5609                sync.conflict = Some(why.clone());
5610                self.state.event("land", why);
5611                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5612                    .await?;
5613                return Ok(());
5614            }
5615            if head != op.head {
5616                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5617            }
5618            tip = op.tip.clone();
5619        }
5620        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5621        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5622        if pending.is_none()
5623            && !conflicted
5624            && fetched_base
5625            && self
5626                .state
5627                .base_sync_recovery
5628                .episodes
5629                .last()
5630                .is_some_and(|e| e.boundary.is_some())
5631            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5632            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5633            && owner.execution_runs().last() == Some(&self.state.id)
5634            && owner.status != crate::queue::TaskStatus::Done
5635            && !owner.fresh_start
5636            && owner.review_branch.is_none()
5637            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5638            && let Some(approval) = owner.resumption.as_ref()
5639            && approval.instruction == owner.instruction
5640            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5641            && git::is_clean(&winner.worktree).await?
5642            && self.state.renew_base_episode(&task, approval, &head, &tip)
5643        {
5644            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5645            self.state.save()?;
5646        }
5647
5648        // Before any rebase, and before a recorded conflict is honoured: a
5649        // branch whose change reached the base under other commit ids has
5650        // nothing to rebase and nothing to conflict with, and a run that
5651        // already stopped on that phantom conflict recovers here on resume.
5652        // `behind == 0` with head == tip is a branch the base has since taken
5653        // in whole, whether or not a conflict was ever recorded: the ancestry
5654        // proof must still run (`classify` ignores a head still on the start
5655        // commit).
5656        if (behind > 0 || conflicted || head == tip)
5657            && self
5658                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5659                .await?
5660        {
5661            return Ok(());
5662        }
5663        if conflicted && self.state.base_recovery_pending() {
5664            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5665            let owner =
5666                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5667            let owner_valid = owner.as_ref().is_ok_and(|t| {
5668                t.execution_runs().last() == Some(&self.state.id)
5669                    && t.status != crate::queue::TaskStatus::Done
5670                    && !t.fresh_start
5671                    && t.review_branch.is_none()
5672            });
5673            let invalidation = if !owner_valid {
5674                Some("latest task ownership changed")
5675            } else if !owner.as_ref().is_ok_and(|t| {
5676                authorization
5677                    .generation
5678                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5679            }) {
5680                Some("owner approval generation changed")
5681            } else if authorization.question.as_ref().is_some_and(|question| {
5682                !crate::base_recovery_control::decision_current(
5683                    &crate::queue::Queue::at(crate::run::home().join("queue")),
5684                    &self.state,
5685                    &authorization.task,
5686                    question,
5687                )
5688            }) {
5689                Some("recovery decision ownership, routing or stop evidence changed")
5690            } else if authorization.request.is_some()
5691                && !owner
5692                    .as_ref()
5693                    .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5694            {
5695                Some("owner decision changed")
5696            } else if let Some(scope) = &authorization.scope {
5697                if scope.tip != tip {
5698                    Some("fetched base changed")
5699                } else if !self
5700                    .state
5701                    .base_sync_recovery
5702                    .episodes
5703                    .last()
5704                    .is_some_and(|e| e.id == scope.episode)
5705                {
5706                    Some("active synchronization interval changed")
5707                } else if !owner
5708                    .as_ref()
5709                    .is_ok_and(|t| t.instruction == scope.instruction)
5710                {
5711                    Some("task instruction changed")
5712                } else {
5713                    None
5714                }
5715            } else {
5716                None
5717            };
5718            if let Some(reason) = invalidation {
5719                let authorization = self
5720                    .state
5721                    .base_sync_recovery
5722                    .authorizations
5723                    .last_mut()
5724                    .unwrap();
5725                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5726                self.state.save()?;
5727                return Ok(());
5728            }
5729            if !fetched_base
5730                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5731                || !git::is_clean(&winner.worktree).await?
5732                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5733            {
5734                return Ok(());
5735            }
5736        }
5737        let recovery = conflicted && self.state.consume_base_recovery(&head);
5738        if conflicted {
5739            // Consumption (including a refused changed HEAD) is durable before Git.
5740            self.state.save()?;
5741            if !recovery {
5742                return Ok(());
5743            }
5744        }
5745
5746        if behind == 0 {
5747            // A fixer-finished rebase moves the branch ref before the
5748            // winner's worktree is told (`sync_to_head` below). A run that
5749            // died in between resumes here with `behind == 0` and a tree still
5750            // holding the pre-rebase files, which review and the gate would
5751            // then read. That state is exactly: HEAD moved off the tip the
5752            // rebase started from, yet the tree is still identical to that
5753            // tip. A tree with edits of its own differs from it, so nothing
5754            // is thrown away.
5755            if let Some(from) = self
5756                .state
5757                .rebase_fixes
5758                .iter()
5759                .rev()
5760                .find_map(|r| r.from.clone())
5761                && from != head
5762                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5763                    .await
5764                    .is_ok_and(|o| o.ok())
5765            {
5766                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5767            }
5768            // An earlier attempt may have rebased the branch locally and died
5769            // before pushing it (only the fresh-rebase arm below pushes).
5770            // Publish it now, so the plain push at PR time is not refused as
5771            // a non-fast-forward. A run already holding a recorded conflict
5772            // never reaches here; that case is out of scope.
5773            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5774            if let Some(why) = &conflict {
5775                self.state.status = RunStatus::Blocked;
5776                self.state.event("land", why.clone());
5777            }
5778            self.state.base_sync = Some(BaseSync {
5779                tip,
5780                behind: 0,
5781                attempts,
5782                conflict,
5783                already_in: None,
5784            });
5785            if pending.is_some() {
5786                self.state.finish_base_operation(Some(&head));
5787            }
5788            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5789                .await?;
5790            self.state.save()?;
5791            return Ok(());
5792        }
5793
5794        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5795            let why = format!(
5796                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5797                 synchronization(s); synchronizing again would only race it",
5798                winner.branch
5799            );
5800            self.state.status = RunStatus::Blocked;
5801            self.state.base_sync = Some(BaseSync {
5802                tip,
5803                behind,
5804                attempts,
5805                conflict: Some(why.clone()),
5806                already_in: None,
5807            });
5808            self.state.event("land", why);
5809            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5810                .await?;
5811            self.state.save()?;
5812            return Ok(());
5813        }
5814
5815        if self.state.foreign_checkout.is_some() {
5816            // Delivery to the operator's checkout is fast-forward only. A
5817            // rebase here would discard its saved head from our ancestry.
5818            let wait = repository_wait(&self.state, None);
5819            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5820            if self.state.base_sync.is_none() {
5821                self.state.base_sync = Some(BaseSync {
5822                    tip: tip.clone(),
5823                    behind,
5824                    attempts: 0,
5825                    conflict: None,
5826                    already_in: None,
5827                });
5828            }
5829            self.state.reserve_base_operation(&head, &tip);
5830            self.state.save()?;
5831            let merged = git::git_raw(
5832                &winner.worktree,
5833                &[
5834                    "-c",
5835                    "core.hooksPath=/dev/null",
5836                    "merge",
5837                    "--no-ff",
5838                    "--no-autostash",
5839                    "--no-edit",
5840                    "-m",
5841                    &format!("Merge {tracking} before verifying"),
5842                    &tip,
5843                ],
5844            )
5845            .await?;
5846            let conflict = if merged.ok() {
5847                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5848                self.state.event(
5849                    "land",
5850                    format!("merged {tracking} into {} before verifying", winner.branch),
5851                );
5852                None
5853            } else {
5854                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5855                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5856                }
5857                let why = format!(
5858                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5859                    winner.branch,
5860                    merged.stderr.chars().take(600).collect::<String>()
5861                );
5862                self.state.status = RunStatus::Blocked;
5863                self.state.event("land", &why);
5864                Some(why)
5865            };
5866            let synced = conflict.is_none();
5867            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5868            self.state
5869                .finish_base_operation(synced.then_some(fresh.as_str()));
5870            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5871            self.state.base_sync = Some(BaseSync {
5872                tip,
5873                behind: if synced { 0 } else { behind },
5874                attempts,
5875                conflict,
5876                already_in: None,
5877            });
5878            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5879                .await?;
5880            self.state.save()?;
5881            return Ok(());
5882        }
5883
5884        self.state.event(
5885            "land",
5886            format!(
5887                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5888                winner.branch
5889            ),
5890        );
5891        self.state.save()?;
5892
5893        // The remote's copy of the branch, read now and only if the fetch
5894        // really succeeded (a stale tracking ref must never pin a lease). It is
5895        // pushed over after a rebase only when it is a commit this branch
5896        // already contains, by ancestry or by patch (an earlier rebase of ours
5897        // that never reached the remote): anything else is somebody else's work.
5898        let branch_tracking = format!("{remote}/{}", winner.branch);
5899        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5900        let remote_tip = if self.state.task_pr_branch().is_none()
5901            && matches!(&fetched_branch, Ok(o) if o.ok())
5902        {
5903            git::rev_parse(&repo, &branch_tracking).await.ok()
5904        } else {
5905            None
5906        };
5907        // A remote tip this branch does not contain is somebody else's work:
5908        // rebasing would leave a local tip that can never be pushed. Stop
5909        // before touching anything and say so.
5910        if let Some(theirs) = &remote_tip
5911            && !git::is_ancestor(&repo, theirs, &head).await
5912            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5913                .await
5914                .is_ok_and(|missing| missing.is_empty())
5915        {
5916            let why = format!(
5917                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5918                 them",
5919                short(theirs),
5920                winner.branch
5921            );
5922            self.state.status = RunStatus::Blocked;
5923            self.state.base_sync = Some(BaseSync {
5924                tip,
5925                behind,
5926                attempts,
5927                conflict: Some(why.clone()),
5928                already_in: None,
5929            });
5930            self.state.event("land", why);
5931            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5932                .await?;
5933            self.state.save()?;
5934            return Ok(());
5935        }
5936
5937        if self.state.base_sync.is_none() {
5938            self.state.base_sync = Some(BaseSync {
5939                tip: tip.clone(),
5940                behind,
5941                attempts: 0,
5942                conflict: None,
5943                already_in: None,
5944            });
5945        }
5946        let scratch = self.state.dir().join("base-sync");
5947        if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5948        {
5949            if error.is::<crate::repo_lock::ResourceBlocked>() {
5950                return Ok(());
5951            }
5952            return Err(error);
5953        }
5954        self.state.reserve_base_operation(&head, &tip);
5955        self.state.save()?;
5956        let rebased =
5957            match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
5958                .await
5959            {
5960                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5961                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5962                Err(e) => Err(e),
5963            };
5964        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5965        let stop_kind = match &rebased {
5966            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5967            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5968            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5969        };
5970        match rebased {
5971            Ok(None) => {
5972                // The branch ref moved, but a worktree that already had it
5973                // checked out (the winner's) was not told; sync its index and
5974                // files before anything reads them.
5975                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5976                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5977                self.state.finish_base_operation(Some(&fresh));
5978                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5979                let mut conflict = None;
5980                if let Some(pinned) = &remote_tip {
5981                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5982                    match pushed {
5983                        Ok(o) if o.ok() => self.state.event(
5984                            "land",
5985                            format!("pushed rebased {} to {remote}", winner.branch),
5986                        ),
5987                        Ok(o) => {
5988                            conflict = Some(format!(
5989                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5990                                winner.branch,
5991                                short(pinned),
5992                                o.stderr.chars().take(600).collect::<String>()
5993                            ));
5994                        }
5995                        Err(e) => {
5996                            conflict = Some(format!(
5997                                "rebased {} locally but could not push it: {e:#}",
5998                                winner.branch
5999                            ));
6000                        }
6001                    }
6002                }
6003                if let Some(why) = &conflict {
6004                    self.state.status = RunStatus::Blocked;
6005                    self.state.event("land", why.clone());
6006                }
6007                self.state.base_sync = Some(BaseSync {
6008                    tip: tip.clone(),
6009                    behind: 0,
6010                    attempts,
6011                    conflict,
6012                    already_in: None,
6013                });
6014                self.state
6015                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
6016            }
6017            Ok(Some(conflict)) => {
6018                self.state.finish_base_operation(None);
6019                let why = format!(
6020                    "{} conflicts with {tracking} and did not rebase: {}",
6021                    winner.branch,
6022                    conflict.chars().take(600).collect::<String>()
6023                );
6024                self.state.status = RunStatus::Blocked;
6025                self.state.base_sync = Some(BaseSync {
6026                    tip,
6027                    behind,
6028                    attempts,
6029                    conflict: Some(why.clone()),
6030                    already_in: None,
6031                });
6032                self.state.event("land", why);
6033            }
6034            // A refused Git write or a busy repository: nothing is wrong with
6035            // the branch. The scratch rebase stays standing and the run
6036            // resumes into it (`base_sync` must not claim a conflict).
6037            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6038                self.state.save()?;
6039                return Ok(());
6040            }
6041            Err(e) => {
6042                self.state.finish_base_operation(None);
6043                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6044                self.state.status = RunStatus::Blocked;
6045                self.state.base_sync = Some(BaseSync {
6046                    tip,
6047                    behind,
6048                    attempts,
6049                    conflict: Some(why.clone()),
6050                    already_in: None,
6051                });
6052                self.state.event("land", why);
6053            }
6054        }
6055        if !self.state.base_sync_blocked() {
6056            self.state.base_sync_recovery.stop = None;
6057        }
6058        self.hold_base_sync_kind(stop_kind).await?;
6059        self.state.save()?;
6060        Ok(())
6061    }
6062
6063    /// Push a branch an earlier attempt rebased locally but never published,
6064    /// pinned to the remote tip read right after a successful fetch. Returns
6065    /// the reason when the run must stop; `None` when there was nothing to do
6066    /// (no remote copy, the same tip, or a remote copy this branch already
6067    /// contains, which the PR-time push fast-forwards) or the push succeeded.
6068    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6069        // Task competitions publish only after the PR text gate, to their
6070        // public head; never resurrect a candidate branch on the remote.
6071        if self.state.task_pr_branch().is_some() {
6072            return None;
6073        }
6074        let repo = self.state.repo.clone();
6075        let remote = self.state.config.merge.remote.clone();
6076        let fetched = git::fetch(&repo, &remote, branch).await;
6077        if !matches!(&fetched, Ok(o) if o.ok()) {
6078            return None;
6079        }
6080        let branch_tracking = format!("{remote}/{branch}");
6081        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6082        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6083            return None;
6084        }
6085        if !crate::reconcile::origin_missing(&repo, head, &theirs)
6086            .await
6087            .is_ok_and(|missing| missing.is_empty())
6088        {
6089            return Some(format!(
6090                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6091                 them",
6092                short(&theirs)
6093            ));
6094        }
6095        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6096            Ok(o) if o.ok() => {
6097                self.state
6098                    .event("land", format!("pushed rebased {branch} to {remote}"));
6099                None
6100            }
6101            Ok(o) => Some(format!(
6102                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6103                 someone may have pushed): {}",
6104                short(&theirs),
6105                o.stderr.chars().take(600).collect::<String>()
6106            )),
6107            Err(e) => Some(format!(
6108                "{branch} is rebased locally but could not be pushed: {e:#}"
6109            )),
6110        }
6111    }
6112
6113    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6114    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6115    /// whether it did.
6116    ///
6117    /// Checked only when the base is ahead of the branch. A failing check is
6118    /// "not proven" - the ordinary rebase path then decides - never a reason to
6119    /// stop the run.
6120    ///
6121    /// The remote copy of the branch is held to the same standard as the local
6122    /// one: if it carries a tip this worktree does not, that tip must itself be
6123    /// proven in the base, or nothing is settled (a pull request would
6124    /// otherwise be closed over commits nobody checked). The pull request is
6125    /// closed *before* the terminal status is saved; if that fails for a
6126    /// reason other than a refusal (no network, a `gh` error) the run is left
6127    /// `Blocked` with the reason as its conflict, which a resume retries -
6128    /// the same recovery a phantom conflict gets.
6129    async fn settle_already_in(
6130        &mut self,
6131        branch: &str,
6132        tip: &str,
6133        head: &str,
6134        attempts: usize,
6135        behind: usize,
6136    ) -> Result<bool> {
6137        let repo = self.state.repo.clone();
6138        let remote = self.state.config.merge.remote.clone();
6139        let start = self.state.base_commit.clone();
6140        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6141            Ok(Some(e)) => e,
6142            Ok(None) => return Ok(false),
6143            Err(e) => {
6144                tracing::warn!("already-in-base check for {branch}: {e:#}");
6145                return Ok(false);
6146            }
6147        };
6148        let mut verified = vec![head.to_owned()];
6149        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6150        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6151        if matches!(&fetched, Ok(o) if o.ok())
6152            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6153            && theirs != head
6154        {
6155            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6156                Ok(Some(_)) => verified.push(theirs),
6157                _ => return Ok(false),
6158            }
6159        }
6160        let base_branch = self.state.base_branch.clone();
6161        let message = format!(
6162            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6163             land",
6164            evidence.names(),
6165            evidence.proof.as_str()
6166        );
6167        let closed =
6168            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6169                .await;
6170        match closed {
6171            Ok(Ok(url)) => self
6172                .state
6173                .event("land", format!("closed {url}: superseded on {base_branch}")),
6174            Ok(Err(why)) => self
6175                .state
6176                .event("land", format!("did not close a pull request: {why}")),
6177            Err(e) => {
6178                let why = format!(
6179                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6180                     not be closed ({e:#}); resume to retry"
6181                );
6182                self.state.status = RunStatus::Blocked;
6183                self.state.base_sync = Some(BaseSync {
6184                    tip: tip.to_owned(),
6185                    behind,
6186                    attempts,
6187                    conflict: Some(why.clone()),
6188                    already_in: None,
6189                });
6190                self.state.event("land", why);
6191                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6192                    .await?;
6193                self.state.save()?;
6194                return Ok(true);
6195            }
6196        }
6197        self.state.status = RunStatus::AlreadyInBase;
6198        self.state.base_sync_recovery.stop = None;
6199        self.state.base_sync = Some(BaseSync {
6200            tip: tip.to_owned(),
6201            behind,
6202            attempts,
6203            conflict: None,
6204            already_in: Some(evidence),
6205        });
6206        self.state.event("land", message);
6207        self.state.save()?;
6208        self.settle_questions();
6209        Ok(true)
6210    }
6211
6212    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6213    /// last landed the winner on, once it has run, else the commit the run
6214    /// branched from.
6215    ///
6216    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6217    /// and `vote` all happen before there is a winner to rebase, so they
6218    /// compare every candidate against the branch point on purpose, and a
6219    /// base that moves after they are already done cannot change an answer
6220    /// they already gave.
6221    fn landing_base(&self) -> String {
6222        self.state
6223            .base_sync
6224            .as_ref()
6225            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6226    }
6227
6228    // ------------------------------------------------------- operator fix
6229
6230    /// Route specific, already-recorded review findings to a fixer for a
6231    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6232    /// entry point.
6233    ///
6234    /// Distinct from `review_loop`'s own fix step in three ways: it never
6235    /// runs a reviewer wave, it never spends review-round budget, and what
6236    /// happened is recorded as an [`OperatorFixRequest`] appended to
6237    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6238    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6239    /// and vote must never be rewritten to look like a manufactured blocking
6240    /// verdict.
6241    ///
6242    /// Only meaningful once review has actually concluded: `Ready` (handed
6243    /// off with findings still open, or simply concluded clean while minor
6244    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6245    /// gate failed). Everything else is refused: a run still in progress
6246    /// should simply be resumed, and a `Merged` run's branch has already
6247    /// landed — reopening *this* run's own record cannot change that, so the
6248    /// answer there is a fresh `magi review <branch>`.
6249    ///
6250    /// A real commit here re-verifies through a fresh, ordinary review-only
6251    /// run on the same branch ([`Self::review`]) rather than reopening this
6252    /// run's own `review_loop`: once any round in this run's history went
6253    /// clean, `review_conclusion` treats that as permanent by design (the
6254    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6255    /// way to force one more genuine reviewer wave out of *this* run without
6256    /// either rewriting history or weakening that guarantee for every other
6257    /// caller. A review-only run costs nothing extra — no implementation, no
6258    /// judging, no vote — and exercises the exact same review → verify →
6259    /// gate → (human) merge path, unmodified.
6260    pub async fn fix_selected(
6261        &mut self,
6262        ids: &[String],
6263        reason: &str,
6264        allow_stale: bool,
6265    ) -> Result<()> {
6266        let reason = reason.trim();
6267        if reason.is_empty() {
6268            bail!("a fix request needs a reason — that is the operator's own record of why");
6269        }
6270        if ids.is_empty() {
6271            bail!("no finding id given");
6272        }
6273        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6274            bail!(
6275                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6276                 has already concluded — can be given a targeted fix. A run still \
6277                 in progress should simply be resumed; a `merged` run's branch has \
6278                 already landed, so its answer is a fresh `magi review <branch>`, \
6279                 not reopening this run's own record",
6280                self.state.id,
6281                self.state.status.as_str()
6282            );
6283        }
6284        // Reassignment shares this claim through its atomic task save. Check
6285        // the canonical run identity inside it, before preflight or any await.
6286        let _claim = FixClaim::acquire(&self.state.dir())?;
6287        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6288        let Some(winner) = self.state.winner().cloned() else {
6289            bail!("run {} has no winning candidate to fix", self.state.id);
6290        };
6291        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6292            bail!(
6293                "branch `{}` no longer exists; this run cannot be extended",
6294                winner.branch
6295            );
6296        }
6297        let home = crate::run::home();
6298        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6299            bail!(
6300                "run {} is currently being worked on by another magi process",
6301                self.state.id
6302            );
6303        }
6304        // Resolve every id before spending anything — an unknown id refuses
6305        // the whole request rather than silently dropping it — and dedup
6306        // while keeping the operator's own order.
6307        let mut seen = BTreeSet::new();
6308        let mut findings = Vec::new();
6309        let mut missing = Vec::new();
6310        for id in ids {
6311            if !seen.insert(id.clone()) {
6312                continue;
6313            }
6314            match self.state.finding(id) {
6315                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6316                    id: f.id.clone(),
6317                    severity: f.severity,
6318                    reviewer_vote: rec.vote,
6319                    round: round.round,
6320                    round_head: round.head.clone(),
6321                    reviewer: rec.reviewer,
6322                    agent: rec.agent.clone(),
6323                    file: f.file.clone(),
6324                    line: f.line,
6325                    title: f.title.clone(),
6326                    detail: f.detail.clone(),
6327                    outcome: OperatorFixOutcome::Pending,
6328                }),
6329                None => missing.push(id.clone()),
6330            }
6331        }
6332        if !missing.is_empty() {
6333            bail!(
6334                "unknown finding id(s): {}; nothing was changed",
6335                missing.join(", ")
6336            );
6337        }
6338
6339        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6340        let stale_details: Vec<(String, String)> = findings
6341            .iter()
6342            .filter(|f| f.round_head != head_at_request)
6343            .map(|f| (f.id.clone(), f.round_head.clone()))
6344            .collect();
6345        let stale = !stale_details.is_empty();
6346        if stale && !allow_stale {
6347            bail!(
6348                "the branch has moved since some finding(s) were raised — {} — now \
6349                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6350                stale_details
6351                    .iter()
6352                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6353                    .collect::<Vec<_>>()
6354                    .join(", "),
6355                short(&head_at_request)
6356            );
6357        }
6358
6359        let request = OperatorFixRequest {
6360            requested_at: Timestamp::now(),
6361            reason: reason.to_owned(),
6362            findings,
6363            head_at_request: head_at_request.clone(),
6364            allow_stale,
6365            stale,
6366            fix: None,
6367            result_head: None,
6368            follow_up_review_run: None,
6369        };
6370        self.state.event(
6371            "fix",
6372            format!(
6373                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6374                request.findings.len(),
6375                request
6376                    .findings
6377                    .iter()
6378                    .map(|f| f.id.as_str())
6379                    .collect::<Vec<_>>()
6380                    .join(", "),
6381            ),
6382        );
6383        // Recorded now, before any worktree work or the fixer call itself —
6384        // and re-saved at each checkpoint below: a crash at any point after
6385        // this (mid fixer call, mid follow-up review) must not lose the fact
6386        // that this was requested, for which findings, and why. Everything
6387        // past this point reads and writes through `request_index` rather
6388        // than a local variable, since `request` itself is moved here.
6389        self.state.operator_fixes.push(request);
6390        self.state.save()?;
6391        let request_index = self.state.operator_fixes.len() - 1;
6392
6393        // A fresh, dedicated worktree for this one call, never the winner's
6394        // own worktree in place: that one may already be gone (folded away),
6395        // and reusing it in place would leave the branch checked out there
6396        // when the follow-up review below tries to check it out again. Freed
6397        // immediately after, either way — but only once confirmed clean:
6398        // `worktree_remove` is a `git worktree remove --force`, which would
6399        // otherwise discard uncommitted work left there by the operator or
6400        // another process before this had a chance to even look at it.
6401        let repo = self.state.repo.clone();
6402        let coordination = self.coordinate(&repo).await?;
6403        if winner.worktree.exists() {
6404            // Lockfiles a rescue commit withheld stay untracked on purpose and
6405            // are already recorded; they are not the operator's work to protect.
6406            let dirty = git::git(
6407                &winner.worktree,
6408                &["status", "--porcelain", "--untracked-files=all"],
6409            )
6410            .await?;
6411            let only_withheld = dirty.lines().all(|l| {
6412                l.strip_prefix("?? ")
6413                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6414            });
6415            if !only_withheld {
6416                bail!(
6417                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6418                     discard them first",
6419                    winner.worktree.display()
6420                );
6421            }
6422            git::worktree_remove(&self.state.repo, &winner.worktree)
6423                .await
6424                .ok();
6425        }
6426        let fix_worktree = self.state.worktree_root().join("operator-fix");
6427        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6428        git::git(
6429            &self.state.repo,
6430            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6431        )
6432        .await
6433        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6434        if !git::is_clean(&fix_worktree).await? {
6435            git::worktree_remove(&self.state.repo, &fix_worktree)
6436                .await
6437                .ok();
6438            bail!(
6439                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6440                winner.branch
6441            );
6442        }
6443        let repo_for_setup = self.state.repo.clone();
6444        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6445            Ok(()) => {
6446                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6447            }
6448            Err(e) => Err(e),
6449        };
6450        if let Err(e) = setup {
6451            git::worktree_remove(&self.state.repo, &fix_worktree)
6452                .await
6453                .ok();
6454            return Err(e);
6455        }
6456
6457        // This explicit operator request retries availability independently
6458        // of execute(); the previous call has already saved and cleaned up.
6459        self.state.resource_block = None;
6460
6461        let run_id = self.state.id.clone();
6462        let prompts = self.state.config.prompts.clone();
6463        let language = self.state.config.graph.language.clone();
6464        let sessions = self.state.config.graph.sessions;
6465        let artifacts = agent::artifacts_dir(&self.state.dir());
6466        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6467            .findings
6468            .iter()
6469            .map(|f| Finding {
6470                id: f.id.clone(),
6471                severity: f.severity,
6472                file: f.file.clone(),
6473                line: f.line,
6474                title: f.title.clone(),
6475                detail: f.detail.clone(),
6476            })
6477            .collect();
6478        let fix_prompt = prompt::operator_fix_in(
6479            &self.state.instruction,
6480            &finding_list,
6481            reason,
6482            &stale_details,
6483            &head_at_request,
6484            &language,
6485            &self.state.config.graph.github_language,
6486        );
6487        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6488        drop(coordination);
6489        let (job, seat, out) = self
6490            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6491                prompt: fix_prompt.clone(),
6492                spec,
6493                seat,
6494                cwd: fix_worktree.clone(),
6495                timeout,
6496                allow_write: true,
6497                sessions,
6498                artifacts: artifacts.clone(),
6499                stem: "operator-fix".to_owned(),
6500                handover: None,
6501            })
6502            .await;
6503        let agent_id = seat.agent.clone();
6504
6505        let mut fix = FixRecord {
6506            agent: agent_id,
6507            addressed: Vec::new(),
6508            rejected: Vec::new(),
6509            notes: String::new(),
6510            committed: false,
6511            failed: None,
6512            duration_ms: 0,
6513            continuation: None,
6514        };
6515        let mut final_seat = seat.clone();
6516        match out {
6517            AgentOutcome::Ok(o) => {
6518                fix.duration_ms = o.duration_ms;
6519                let parsed = verdict::extract_json::<FixReport>(&o.text);
6520                let incomplete_reason = match &parsed {
6521                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6522                        "the reply parsed, but it reported a command whose own CLI \
6523                         never confirmed an exit status"
6524                            .to_owned(),
6525                    ),
6526                    Ok(_) => None,
6527                    Err(e) => Some(e.to_string()),
6528                };
6529                match incomplete_reason {
6530                    None => {
6531                        let report = parsed.expect("checked Ok above");
6532                        fix.addressed = report.addressed;
6533                        fix.rejected = report.rejected;
6534                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6535                    }
6536                    Some(reason) => {
6537                        let (resumed_seat, resolved, failure, cont) = self
6538                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6539                            .await;
6540                        fix.duration_ms += cont.cumulative_wait_ms;
6541                        fix.continuation = Some(cont);
6542                        final_seat = resumed_seat;
6543                        match resolved {
6544                            Some(report) => {
6545                                fix.addressed = report.addressed;
6546                                fix.rejected = report.rejected;
6547                                fix.notes =
6548                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6549                            }
6550                            None => fix.failed = failure,
6551                        }
6552                    }
6553                }
6554            }
6555            AgentOutcome::Dropped(o) => {
6556                fix.duration_ms = o.duration_ms;
6557                let why = o
6558                    .dropped
6559                    .as_ref()
6560                    .map(|d| d.why.as_str())
6561                    .unwrap_or("the CLI ended the stream without delivering its answer");
6562                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6563            }
6564            AgentOutcome::Quota(o) => {
6565                self.state.quota.push(QuotaLoss::from_report(
6566                    final_seat.key.clone(),
6567                    "fix",
6568                    o.quota.as_ref(),
6569                ));
6570                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6571            }
6572            AgentOutcome::Failed(e) => fix.failed = Some(e),
6573            AgentOutcome::ResourceBlocked(e) => {
6574                fix.failed = Some(format!("the fixer was not started: {e}"));
6575            }
6576        }
6577        // Even a deferred call must rescue edits, release the worktree and
6578        // finalize its record below so an explicit `magi fix` can retry.
6579        if fix.continuation.is_none() {
6580            fix.continuation = Some(ContinuationRecord::not_needed());
6581        }
6582        self.state.seats.insert(final_seat.key.clone(), final_seat);
6583
6584        let rescue_message = format!(
6585            "magi: operator-selected fix ({}) (uncommitted work)",
6586            self.state.operator_fixes[request_index]
6587                .findings
6588                .iter()
6589                .map(|f| f.id.as_str())
6590                .collect::<Vec<_>>()
6591                .join(", ")
6592        );
6593        let repo = self.state.repo.clone();
6594        let coordination = self.coordinate(&repo).await?;
6595        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6596            self.state.note_withheld("fix", &r.withheld);
6597        }
6598        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6599        fix.committed = after != head_at_request;
6600        git::worktree_remove(&self.state.repo, &fix_worktree)
6601            .await
6602            .ok();
6603        drop(coordination);
6604
6605        self.state.event(
6606            "fix",
6607            match &fix.failed {
6608                Some(reason) => format!(
6609                    "operator fix: adoption report was lost ({reason}); {}",
6610                    if fix.committed {
6611                        "committed"
6612                    } else {
6613                        "NO new commit"
6614                    }
6615                ),
6616                None => format!(
6617                    "operator fix: {} addressed, {} rejected, {}",
6618                    fix.addressed.len(),
6619                    fix.rejected.len(),
6620                    if fix.committed {
6621                        "committed"
6622                    } else {
6623                        "NO new commit"
6624                    }
6625                ),
6626            },
6627        );
6628
6629        // Every selected finding gets an outcome — never left `Pending` once
6630        // the fixer's own turn is over. A report that never came back at all
6631        // marks every one of them `Unreported`, not silently "not addressed":
6632        // quota, a dropped stream, or an exhausted continuation are gaps in
6633        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6634        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6635        for f in &mut self.state.operator_fixes[request_index].findings {
6636            f.outcome = if fix.failed.is_some() {
6637                OperatorFixOutcome::Unreported
6638            } else if fix.addressed.contains(&f.id) {
6639                OperatorFixOutcome::Addressed
6640            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6641                OperatorFixOutcome::Rejected { why: r.why.clone() }
6642            } else {
6643                OperatorFixOutcome::Unreported
6644            };
6645        }
6646
6647        let committed = fix.committed;
6648        if committed {
6649            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6650        }
6651        self.state.operator_fixes[request_index].fix = Some(fix);
6652        // Saved again now that the fixer's own outcome is final, on top of
6653        // the save right after the request was first pushed above.
6654        self.state.save()?;
6655        if self.state.resource_block.is_some() {
6656            return self.stop_for_resource();
6657        }
6658
6659        if committed {
6660            self.state.event(
6661                "fix",
6662                format!(
6663                    "operator fix committed {}; opening a follow-up review-only run",
6664                    short(&after)
6665                ),
6666            );
6667            // The operator asked for the fix, and the follow-up serves whatever
6668            // task the run it follows served.
6669            let origin =
6670                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6671            match Self::review(
6672                &self.state.repo,
6673                &winner.branch,
6674                self.state.config.clone(),
6675                origin,
6676            )
6677            .await
6678            {
6679                Ok(mut follow_up) => {
6680                    follow_up.state.event(
6681                        "start",
6682                        format!(
6683                            "requested by an operator fix on run {} for finding(s) {}",
6684                            self.state.id,
6685                            self.state.operator_fixes[request_index]
6686                                .findings
6687                                .iter()
6688                                .map(|f| f.id.as_str())
6689                                .collect::<Vec<_>>()
6690                                .join(", "),
6691                        ),
6692                    );
6693                    follow_up.state.save()?;
6694                    let follow_up_id = follow_up.state.id.clone();
6695                    // The follow-up is a run like any other: it belongs to the
6696                    // task of the run it follows, or to one filed for it.
6697                    let adopted = match crate::direct::adopt(
6698                        &follow_up.state,
6699                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6700                    ) {
6701                        Ok(a) => a,
6702                        Err(e) => {
6703                            // An ownerless run must not spend agent calls; it
6704                            // stays saved, and `magi run --resume` adopts it.
6705                            self.state.event(
6706                                "fix",
6707                                format!(
6708                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6709                                ),
6710                            );
6711                            self.state.save()?;
6712                            return Ok(());
6713                        }
6714                    };
6715                    let executed = follow_up.execute().await;
6716                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6717                    if let Some(a) = adopted {
6718                        a.finish(&follow_up.state, executed);
6719                    }
6720                    if let Some(e) = failure {
6721                        self.state.event(
6722                            "fix",
6723                            format!(
6724                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6725                            ),
6726                        );
6727                    }
6728                    self.state.operator_fixes[request_index].follow_up_review_run =
6729                        Some(follow_up_id);
6730                }
6731                Err(e) => {
6732                    self.state.event(
6733                        "fix",
6734                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6735                    );
6736                }
6737            }
6738            self.state.save()?;
6739        }
6740
6741        Ok(())
6742    }
6743
6744    // --------------------------------------------------------------- review
6745
6746    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6747    /// names, in order, each at most once, moving on only when the call
6748    /// advances (an error, a quota hit or nothing usable - the same decision
6749    /// point `agent::chain_advances` is for the other chained roles).
6750    ///
6751    /// Each agent gets its own seat from `Runner::seat`: the same agent
6752    /// continues its conversation (the winner's own implementer seat when it
6753    /// is the winner's author, now that the competition is over), another
6754    /// takes a fresh one so the full prompt is sent again. A handover is
6755    /// recorded under `node`, which is what makes the fallback stick for the
6756    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6757    /// is returned, so an exhausted chain reads exactly like a single failed
6758    /// fixer: one quota loss, the same wording, the same refund. An earlier
6759    /// attempt's edits are left in the tree and judged, with the final
6760    /// attempt's, by what git says afterwards.
6761    ///
6762    /// Returns the job that produced the outcome, for `continue_fix_report`.
6763    async fn ask_fixer(
6764        &mut self,
6765        winner: &Candidate,
6766        node: &'static str,
6767        round: Option<usize>,
6768        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6769    ) -> (SeatJob, SeatState, AgentOutcome) {
6770        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6771        self.ask_fixer_among(attempts, node, round, None, build)
6772            .await
6773    }
6774
6775    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6776    /// worktree and a commit: before the next agent is asked after one that
6777    /// did not satisfy the call, the worktree is put back on that commit, so
6778    /// a failed agent's partial edits are not carried into the next one.
6779    async fn ask_fixer_among(
6780        &mut self,
6781        attempts: Vec<(AgentSpec, String)>,
6782        node: &'static str,
6783        round: Option<usize>,
6784        restore: Option<(&Path, &str)>,
6785        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6786    ) -> (SeatJob, SeatState, AgentOutcome) {
6787        let run_id = self.state.id.clone();
6788        let prompts = self.state.config.prompts.clone();
6789        let cache = self.state.config.cache_dir();
6790        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6791        let mut last = None;
6792        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6793            let seat = self.seat(&key, &spec.id);
6794            let mut job = build(spec.clone(), seat);
6795            if i > 0 {
6796                job.stem = format!("{}-{}", job.stem, spec.id);
6797            }
6798            let ctx = WaveCtx {
6799                carry_seats: false,
6800                pauses: Some((&self.pause, &self.interrupt)),
6801                run: &run_id,
6802                node,
6803                prompts: &prompts,
6804                cache: cache.as_deref(),
6805                round,
6806            };
6807            let (seat, out) =
6808                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6809            // A resource block never advances the chain: the chosen agent
6810            // was not asked, so it must be the one asked again on resume.
6811            let advances = match &out {
6812                AgentOutcome::Ok(o) => agent::output_advances(o),
6813                AgentOutcome::ResourceBlocked(_) => false,
6814                _ => true,
6815            };
6816            if let Some(next) = ids.get(i + 1)
6817                && advances
6818            {
6819                self.state.seats.insert(seat.key.clone(), seat.clone());
6820                let class =
6821                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6822                let reason = match &out {
6823                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6824                    other => fail_reason(other),
6825                };
6826                // A failed agent's edits must not reach the next one, and a
6827                // restore that did not work ends the chain: the call fails
6828                // and the caller's own (confirmed) restore decides.
6829                if let Some((wt, rev)) = restore
6830                    && let Err(e) = self.graft_restore(wt, rev).await
6831                {
6832                    let why =
6833                        format!("could not restore the worktree before the next fixer: {e:#}");
6834                    tracing::warn!("{why}");
6835                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6836                        AgentOutcome::ResourceBlocked(why)
6837                    } else {
6838                        AgentOutcome::Failed(why)
6839                    };
6840                    last = Some((job, seat, out));
6841                    break;
6842                }
6843                record_handover(
6844                    &mut self.state,
6845                    node,
6846                    &seat.key,
6847                    &spec.id,
6848                    next,
6849                    &class,
6850                    &reason,
6851                );
6852                continue;
6853            }
6854            last = Some((job, seat, out));
6855            break;
6856        }
6857        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6858        if let AgentOutcome::Quota(output) = &out
6859            && node != "graft"
6860        {
6861            // Graft is best effort: its caller records the loss and restores
6862            // the pre-graft tree, then proceeds to review and gate.
6863            // Every fallback was tried. Machine outages cannot be repaired by
6864            // spending a fix round; use each caller's resumable resource path.
6865            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6866            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6867            self.state.quota.push(loss);
6868            self.state.seats.insert(seat.key.clone(), seat.clone());
6869            self.state.status = RunStatus::Blocked;
6870            return (
6871                job,
6872                seat,
6873                AgentOutcome::ResourceBlocked(
6874                    "fixer unavailable; restore agent availability and resume".into(),
6875                ),
6876            );
6877        }
6878        (job, seat, out)
6879    }
6880
6881    /// The one repair a verification-only failure earns (see
6882    /// [`crate::recovery`]), before the first review round of a run that took
6883    /// a branch over.
6884    ///
6885    /// Saved as started *before* the fixer is called, so a resume never
6886    /// repairs twice; a start found on re-entry is rolled back to the commit
6887    /// it began from. A failed or unusable fixer leaves the tree as it was and
6888    /// the review goes on over it, so the first round's own verification
6889    /// reports the failure again. The failure is never carried into a round as
6890    /// a result: verification is re-run on whatever commit this leaves.
6891    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6892        use crate::recovery::{Mode, Repair};
6893        if !self.state.reviews.is_empty() {
6894            return Ok(());
6895        }
6896        let Some(mut rec) = self.state.recovery.clone() else {
6897            return Ok(());
6898        };
6899        if rec.mode != Mode::VerifyRepair {
6900            return Ok(());
6901        }
6902        let wt = winner.worktree.clone();
6903        let mut head = git::rev_parse(&wt, "HEAD").await?;
6904        match rec.repair.clone() {
6905            Repair::Done { .. } => return Ok(()),
6906            Repair::WaitingForRestore { before } => {
6907                self.graft_restore(&wt, &before).await?;
6908                head = before;
6909                rec.repair = Repair::NotStarted;
6910                self.state.recovery = Some(rec.clone());
6911                self.state.save()?;
6912            }
6913            Repair::Started { before } => {
6914                // Interrupted: the unfinished edits are not trusted.
6915                // An unconfirmed rollback blocks the run with `Started` kept,
6916                // so a resume retries it; the tree is never reviewed as is.
6917                if let Err(e) = self.graft_restore(&wt, &before).await {
6918                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6919                        return Err(e);
6920                    }
6921                    return self
6922                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6923                        .await;
6924                }
6925                rec.repair = Repair::Done {
6926                    before: before.clone(),
6927                    after: before,
6928                    committed: false,
6929                    failed: Some("interrupted; rolled back".to_owned()),
6930                };
6931                self.state
6932                    .event("recovery", "the repair was interrupted and rolled back");
6933                self.state.recovery = Some(rec);
6934                self.state.save()?;
6935                return Ok(());
6936            }
6937            Repair::NotStarted => {}
6938        }
6939        if crate::recovery::recheck(&mut rec, &head) {
6940            self.state.event("recovery", rec.summary());
6941            self.state.recovery = Some(rec);
6942            self.state.save()?;
6943            return Ok(());
6944        }
6945        rec.repair = Repair::Started {
6946            before: head.clone(),
6947        };
6948        self.state.recovery = Some(rec.clone());
6949        self.state.save()?;
6950
6951        let language = self.state.config.graph.language.clone();
6952        let sessions = self.state.config.graph.sessions;
6953        let artifacts = agent::artifacts_dir(&self.state.dir());
6954        let fix_prompt = prompt::recovery_fix(
6955            &self.state.instruction,
6956            &rec,
6957            self.state.config.graph.review_rounds,
6958            &language,
6959        );
6960        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6961        self.state.event(
6962            "recovery",
6963            format!(
6964                "repair: the fixer starts from the failed verification on {}",
6965                short(&head)
6966            ),
6967        );
6968        let (_job, seat, out) = self
6969            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6970                prompt: fix_prompt.clone(),
6971                spec,
6972                seat,
6973                cwd: wt.clone(),
6974                timeout,
6975                allow_write: true,
6976                sessions,
6977                artifacts: artifacts.clone(),
6978                stem: "fix-recovery".to_owned(),
6979                handover: None,
6980            })
6981            .await;
6982        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6983            rec.repair = Repair::WaitingForRestore { before: head };
6984            self.state.recovery = Some(rec);
6985            return self.stop_for_resource();
6986        }
6987        let failed = match &out {
6988            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6989            AgentOutcome::Quota(o) => {
6990                self.state.quota.push(QuotaLoss::from_report(
6991                    seat.key.clone(),
6992                    "fix",
6993                    o.quota.as_ref(),
6994                ));
6995                Some("rate limited (quota); the fixer could not run".to_owned())
6996            }
6997            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6998            AgentOutcome::Failed(e) => Some(e.clone()),
6999        };
7000        self.state.seats.insert(seat.key.clone(), seat);
7001        let coordination = self.coordinate(&wt).await?;
7002        let failed = match failed {
7003            // The agent's partial edits are not kept.
7004            Some(why) => {
7005                if let Err(e) = git::restore_to(&wt, &head).await {
7006                    return self
7007                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7008                        .await;
7009                }
7010                Some(why)
7011            }
7012            None => {
7013                if let Ok(r) = git::rescue_commit(
7014                    &wt,
7015                    "magi: recovery repair of the failed verification (uncommitted work)",
7016                )
7017                .await
7018                {
7019                    self.state.note_withheld("fix", &r.withheld);
7020                }
7021                None
7022            }
7023        };
7024        let after = git::rev_parse(&wt, "HEAD").await?;
7025        drop(coordination);
7026        self.state.event(
7027            "recovery",
7028            match &failed {
7029                Some(why) => format!("repair: nothing kept ({why})"),
7030                None if after != head => {
7031                    format!("repair: committed {}", short(&after))
7032                }
7033                None => "repair: the fixer made no commit".to_owned(),
7034            },
7035        );
7036        rec.repair = Repair::Done {
7037            before: head.clone(),
7038            committed: after != head,
7039            after,
7040            failed,
7041        };
7042        self.state.recovery = Some(rec);
7043        self.state.save()?;
7044        Ok(())
7045    }
7046
7047    /// A repair whose rollback could not be confirmed: the run is blocked and
7048    /// resumable, and the unconfirmed state is never reviewed.
7049    async fn stop_recovery(
7050        &mut self,
7051        mut rec: crate::recovery::RecoveryRecord,
7052        head: &str,
7053        why: String,
7054    ) -> Result<()> {
7055        // `Started` stays recorded: a resume retries the restore.
7056        rec.repair = crate::recovery::Repair::Started {
7057            before: head.to_owned(),
7058        };
7059        self.state.recovery = Some(rec);
7060        self.state
7061            .event("recovery", format!("repair could not be undone: {why}"));
7062        self.state.status = RunStatus::Blocked;
7063        self.state.save()?;
7064        bail!("recovery repair could not be rolled back: {why}")
7065    }
7066
7067    async fn review_loop(&mut self) -> Result<()> {
7068        if self.hold_base_sync().await? {
7069            return Ok(());
7070        }
7071        loop {
7072            if self
7073                .state
7074                .base_sync
7075                .as_ref()
7076                .is_some_and(|s| s.conflict.is_some())
7077            {
7078                return Ok(());
7079            }
7080            if self.catchup_repairable() {
7081                let record = self.state.reviews.last().expect("repairable round").clone();
7082                let winner = self
7083                    .state
7084                    .winner()
7085                    .cloned()
7086                    .context("catch-up repair has no winner")?;
7087                if !self.fix_review_round(&winner, record).await? {
7088                    return Ok(());
7089                }
7090            }
7091            self.review_loop_pass().await?;
7092            if !self.catchup_repairable() {
7093                return Ok(());
7094            }
7095        }
7096    }
7097
7098    /// Derive an owed catch-up repair from durable round evidence, including
7099    /// reentry after a crash before the fixer was dispatched.
7100    fn catchup_repairable(&self) -> bool {
7101        self.state.reviews.len() < self.state.config.graph.review_rounds
7102            && self.state.reviews.last().is_some_and(|r| {
7103                !r.clean
7104                    && r.blocking == 0
7105                    && r.fix.is_none()
7106                    && r.e2e_status() == E2eStatus::Failed
7107            })
7108    }
7109
7110    async fn review_loop_pass(&mut self) -> Result<()> {
7111        // A base that would not rebase is a person's decision, not a review
7112        // round: nothing here would change the answer, and reviewers and a
7113        // fixer would be spending real budget on a tree that cannot land
7114        // regardless of what they find.
7115        if self
7116            .state
7117            .base_sync
7118            .as_ref()
7119            .is_some_and(|s| s.conflict.is_some())
7120        {
7121            return Ok(());
7122        }
7123        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7124        // agent files with `magi task add` name the run that paid for it. The
7125        // prompt overlay is cloned alongside it because the waves borrow it
7126        // while `self` is mutably borrowed by the node's own bookkeeping.
7127        let run_id = self.state.id.clone();
7128        let prompts = self.state.config.prompts.clone();
7129        let Some(winner) = self.state.winner().cloned() else {
7130            return Ok(());
7131        };
7132        let granted_review = self
7133            .state
7134            .repair_grants
7135            .last()
7136            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7137            .cloned();
7138        let fresh_review = granted_review
7139            .as_ref()
7140            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7141        let max_rounds = granted_review
7142            .as_ref()
7143            .map_or(self.state.config.graph.review_rounds, |g| {
7144                g.review_start.saturating_add(1)
7145            });
7146        // A clean round, an exhausted round budget, or a stalled tree (see
7147        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7148        // they are recorded — recomputed here, not read off `status`, so a
7149        // reentry into a run that already stopped restates the identical
7150        // verdict instead of silently handing back whatever an earlier node
7151        // in this same walk clobbered `status` to (a solo-candidate
7152        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7153        // below runs an empty range once the budget is spent, and would
7154        // otherwise fall through without touching `status` at all.
7155        if let Some(status) =
7156            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7157        {
7158            // A reentry after a crash between the last round's save and
7159            // `stop_reviewing` reaches the hand-off here, not there.
7160            if status == RunStatus::Gating {
7161                self.record_contested_handoff();
7162            }
7163            self.state.status = status;
7164            self.state.save()?;
7165            return Ok(());
7166        }
7167        self.state.status = RunStatus::Reviewing;
7168        // A last recorded round whose own verification never resolved
7169        // (`ResourceBlocked` — the shared build cache, not the patch) is
7170        // never a concluded round, whatever the round budget says: starting
7171        // a fresh round on top of it would spend a whole new reviewer wave
7172        // re-reading an unchanged patch instead of just retrying the one
7173        // check that actually needs it, and once the budget is spent the
7174        // loop below has nothing left to do at all (its range is empty).
7175        // Retry that check directly instead, exactly the same retry
7176        // `stop_reviewing` already does for its own catch-up case.
7177        if !fresh_review
7178            && self.state.reviews.last().is_some_and(|r| {
7179                r.e2e_status() == E2eStatus::ResourceBlocked
7180                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7181            })
7182        {
7183            let shell = self.state.config.shell();
7184            return self
7185                .stop_unresolved_review(
7186                    "the last round's own verification never resolved",
7187                    &shell,
7188                    &winner.worktree,
7189                )
7190                .await;
7191        }
7192
7193        if granted_review.is_none() {
7194            self.recovery_repair(&winner).await?;
7195        }
7196        if self.resource_halt()? {
7197            return Ok(());
7198        }
7199
7200        let repo = self.state.repo.clone();
7201        let root = self.state.worktree_root();
7202        let language = self.state.config.graph.language.clone();
7203        let sessions = self.state.config.graph.sessions;
7204        let artifacts = self.merge_repair_artifacts();
7205        let base = self.landing_base();
7206        let base_short = short(&base);
7207        let reviewers = self.roles.reviewers.clone();
7208        let shell = self.state.config.shell();
7209
7210        for round in (self.state.reviews.len() + 1)..=max_rounds {
7211            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7212            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7213            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7214            // The prior round's own record, already persisted — never a
7215            // hand-carried variable of just its failing output: that is
7216            // exactly what let a round's e2e result drift out of sync with
7217            // which commit it was actually about (see `SCHEMA`'s doc for
7218            // schema 8). Judged against `head`, the commit reviewers are
7219            // about to look at now, so the summary always reads as "an
7220            // earlier head" here — this round's own patch has not been
7221            // checked yet.
7222            let prev_verification = self
7223                .state
7224                .reviews
7225                .last()
7226                .and_then(|r| r.verification_summary(&head));
7227            // Only the first round of a run that took a branch over: later
7228            // rounds have this run's own history.
7229            let prior_section = match (&self.state.recovery, round) {
7230                (Some(rec), 1) if self.state.reviews.is_empty() => {
7231                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7232                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7233                            Some(crate::recovery::head_of(
7234                                &git::diff(&winner.worktree, reviewed, "HEAD")
7235                                    .await
7236                                    .unwrap_or_default(),
7237                                crate::recovery::SINCE_DIFF_MAX,
7238                            ))
7239                        }
7240                        _ => None,
7241                    };
7242                    Some(prompt::recovery_review(rec, since.as_deref()))
7243                }
7244                _ => None,
7245            };
7246
7247            // Each reviewer gets its own detached checkout of exactly this
7248            // commit: nobody can perturb the winner's tree, and the fixer can
7249            // keep working without racing a reviewer.
7250            let mut jobs = Vec::new();
7251            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7252                self.coordinate(&repo).await?
7253            } else {
7254                None
7255            };
7256            for (r, spec) in reviewers.iter().cloned().enumerate() {
7257                let wt = root.join(format!("review-{}", r + 1));
7258                // `clean -fdx` in a reset wipes setup's products, so every
7259                // round sets the seat up again.
7260                if wt.exists() {
7261                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7262                } else {
7263                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7264                        None
7265                    } else {
7266                        self.coordinate(&repo).await?
7267                    };
7268                    git::worktree_add_detached(&repo, &wt, &head).await?;
7269                }
7270                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7271                    None
7272                } else {
7273                    self.coordinate(&repo).await?
7274                };
7275                self.setup_seat_worktree(&repo, &wt, false).await?;
7276                let seat_key = format!("review-{}", r + 1);
7277                // The seat starts the round on whoever answered it last, not
7278                // on the agent the spec names, so a failure is not re-paid.
7279                let spec = pick_start_spec(
7280                    &self.roles.reviewer_roster,
7281                    spec,
7282                    self.state.seat_history.get(&seat_key),
7283                );
7284                let seat = self.seat(&seat_key, &spec.id);
7285                jobs.push(SeatJob {
7286                    prompt: prompt::review(&prompt::ReviewCtx {
7287                        instruction: &self.state.instruction,
7288                        branch: &winner.branch,
7289                        base_short: &base_short,
7290                        stat: &stat,
7291                        patch: &patch,
7292                        verification: prev_verification.as_ref(),
7293                        prior: prior_section.as_deref(),
7294                        reviewers: reviewers.len(),
7295                        round,
7296                        rounds: max_rounds,
7297                        // A review-only run has no rankings, so nothing
7298                        // competed for this patch and the reviewer is told so.
7299                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7300                        lens: Lens::for_seat(r),
7301                        language: &language,
7302                        github_language: &self.state.config.graph.github_language,
7303                    }),
7304                    spec,
7305                    seat,
7306                    cwd: wt,
7307                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7308                    allow_write: false,
7309                    sessions,
7310                    artifacts: artifacts.clone(),
7311                    stem: format!("review-{round}-{}", r + 1),
7312                    handover: None,
7313                });
7314            }
7315
7316            drop(coordination);
7317            self.state.event(
7318                "review",
7319                format!(
7320                    "round {round}: {} reviewers on {}",
7321                    jobs.len(),
7322                    short(&head)
7323                ),
7324            );
7325            let mut quota_losses = Vec::new();
7326            let review_retries = self.state.config.graph.retries;
7327            let review_cache = self.state.config.cache_dir();
7328            let ctx = WaveCtx {
7329                carry_seats: true,
7330                pauses: Some((&self.pause, &self.interrupt)),
7331                run: &run_id,
7332                node: "review",
7333                prompts: &prompts,
7334                cache: review_cache.as_deref(),
7335                round: Some(round),
7336            };
7337            let results = ask_json_wave::<Review>(
7338                jobs,
7339                Arc::clone(&self.sem),
7340                review_retries,
7341                &self.roles.reviewer_roster,
7342                &ctx,
7343                &mut quota_losses,
7344                &mut self.state,
7345                &|_: &Review| Ok(()),
7346            )
7347            .await;
7348            if self.state.resource_block.is_some() {
7349                return self.stop_for_resource();
7350            }
7351            // Counted before the move below: how many of *this* round's
7352            // reviewer seats were lost to their own rate limit, as opposed to
7353            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7354            let round_quota_missing = quota_losses.len();
7355            self.state.quota.extend(quota_losses);
7356
7357            let mut records = Vec::new();
7358            let mut all_findings = Vec::new();
7359            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7360                let agent_id = seat.agent.clone();
7361                self.state.seats.insert(seat.key.clone(), seat);
7362                let mut record = ReviewRecord {
7363                    reviewer: r + 1,
7364                    agent: agent_id,
7365                    summary: String::new(),
7366                    findings: Vec::new(),
7367                    vote: None,
7368                    failed: None,
7369                    duration_ms: 0,
7370                    // Set for both outcomes: `failed: Some(_)` with
7371                    // `attempts > 0` is a seat every retry still lost, not a
7372                    // recovered one — only `failed: None` with `attempts > 0`
7373                    // reads as "answered after a nudge" (see this field's own
7374                    // doc).
7375                    attempts,
7376                };
7377                match res {
7378                    Ok((review, out)) => {
7379                        // Sanitized here, at the point every other piece of
7380                        // agent prose in this file is (candidate summaries,
7381                        // deliberation turns, vote reasons): a reviewer's own
7382                        // words are the one thing about it that could name
7383                        // it, and reconsideration below broadcasts this same
7384                        // summary and these same findings to every other
7385                        // seat on the panel.
7386                        record.summary =
7387                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7388                        record.vote = Some(review.vote);
7389                        record.duration_ms = out.duration_ms;
7390                        for (n, mut f) in review.findings.into_iter().enumerate() {
7391                            // ids are magi's, never the agent's: the fixer's
7392                            // adoption report is keyed by them.
7393                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7394                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7395                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7396                            // `file` is agent-supplied prose too, never
7397                            // checked against the real tree — the same
7398                            // exposure `title`/`detail` above have, just in
7399                            // a field easy to forget because it looks like a
7400                            // path rather than free text.
7401                            f.file = f
7402                                .file
7403                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7404                            all_findings.push(f.clone());
7405                            record.findings.push(f);
7406                        }
7407                        self.state.event(
7408                            "review",
7409                            format!(
7410                                "round {round}: reviewer {} voted {} with {} finding(s)",
7411                                r + 1,
7412                                review.vote.label(),
7413                                record.findings.len()
7414                            ),
7415                        );
7416                    }
7417                    Err(e) => {
7418                        record.failed = Some(e.to_string());
7419                        self.state.event(
7420                            "review",
7421                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7422                        );
7423                    }
7424                }
7425                records.push(record);
7426            }
7427
7428            // Tally the round's votes and, if they split, spend the one
7429            // round of reconsideration the split -> deliberate -> revote
7430            // shape `judge`/`vote` use for the panel, sized down to what a
7431            // read-only review round can afford: one round, and a revote
7432            // rather than an argument, because the panel already wrote its
7433            // reasoning down as findings the first time around.
7434            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7435            let vote_split =
7436                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7437            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7438            if vote_split {
7439                self.state.event(
7440                    "review",
7441                    format!(
7442                        "round {round}: votes split ({}) — one round of reconsideration",
7443                        initial_votes
7444                            .iter()
7445                            .map(|v| v.label())
7446                            .collect::<Vec<_>>()
7447                            .join(", ")
7448                    ),
7449                );
7450                // Seats read every seat's findings and votes, still numbered
7451                // and never named — the same anonymity `review` itself keeps.
7452                let panel: Vec<ReviewSeatReport<'_>> = records
7453                    .iter()
7454                    .filter_map(|r| {
7455                        r.vote.map(|vote| ReviewSeatReport {
7456                            reviewer: r.reviewer,
7457                            vote,
7458                            summary: &r.summary,
7459                            findings: &r.findings,
7460                        })
7461                    })
7462                    .collect();
7463
7464                let mut jobs = Vec::new();
7465                let mut seats_at = Vec::new();
7466                for (r, spec) in reviewers.iter().cloned().enumerate() {
7467                    // A seat with no initial vote has nothing to reconsider
7468                    // from and stays absent, the same as it stayed absent
7469                    // from `panel` above.
7470                    if records[r].vote.is_none() {
7471                        continue;
7472                    }
7473                    let wt = root.join(format!("review-{}", r + 1));
7474                    let seat_key = format!("review-{}", r + 1);
7475                    let spec = self.occupant(&seat_key, spec);
7476                    let seat = self.seat(&seat_key, &spec.id);
7477                    // A seat with no live session has already forgotten the
7478                    // initial review's prompt — restate the patch it is
7479                    // voting on, the same as `deliberate`/`vote` do for a
7480                    // judge in the same position.
7481                    // The panel already carries this seat's own review and
7482                    // vote, so restating the patch makes the prompt whole for
7483                    // a seat handed to another agent.
7484                    let build = |with_patch: bool| {
7485                        prompt::review_reconsider(&ReviewReconsiderCtx {
7486                            instruction: &self.state.instruction,
7487                            reviewer: r + 1,
7488                            lens: Lens::for_seat(r),
7489                            panel: &panel,
7490                            patch: with_patch.then_some(ReviewPatch {
7491                                branch: &winner.branch,
7492                                base_short: &base_short,
7493                                stat: &stat,
7494                                patch: &patch,
7495                            }),
7496                            round,
7497                            rounds: max_rounds,
7498                            language: &language,
7499                        })
7500                    };
7501                    let full = build(true);
7502                    let prompt = if has_context(&spec, &seat, sessions) {
7503                        build(false)
7504                    } else {
7505                        full.clone()
7506                    };
7507                    jobs.push(SeatJob {
7508                        prompt,
7509                        spec,
7510                        seat,
7511                        cwd: wt,
7512                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7513                        allow_write: false,
7514                        sessions,
7515                        artifacts: artifacts.clone(),
7516                        stem: format!("review-{round}-reconsider-{}", r + 1),
7517                        handover: Some(full),
7518                    });
7519                    seats_at.push(r);
7520                }
7521
7522                let mut recon_quota_losses = Vec::new();
7523                let recon_cache = self.state.config.cache_dir();
7524                let recon_ctx = WaveCtx {
7525                    carry_seats: true,
7526                    pauses: Some((&self.pause, &self.interrupt)),
7527                    run: &run_id,
7528                    node: "review",
7529                    prompts: &prompts,
7530                    cache: recon_cache.as_deref(),
7531                    round: Some(round),
7532                };
7533                let recon_results = ask_json_wave::<ReviewRevote>(
7534                    jobs,
7535                    Arc::clone(&self.sem),
7536                    review_retries,
7537                    &self.roles.reviewer_roster,
7538                    &recon_ctx,
7539                    &mut recon_quota_losses,
7540                    &mut self.state,
7541                    &|_: &ReviewRevote| Ok(()),
7542                )
7543                .await;
7544                if self.state.resource_block.is_some() {
7545                    return self.stop_for_resource();
7546                }
7547                self.state.quota.extend(recon_quota_losses);
7548
7549                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7550                    let agent_id = seat.agent.clone();
7551                    self.state.seats.insert(seat.key.clone(), seat);
7552                    let mut rec = ReviewRevoteRecord {
7553                        reviewer: r + 1,
7554                        agent: agent_id,
7555                        vote: None,
7556                        reason: String::new(),
7557                        failed: None,
7558                    };
7559                    match res {
7560                        Ok((rv, _)) => {
7561                            rec.vote = Some(rv.vote);
7562                            rec.reason =
7563                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7564                            self.state.event(
7565                                "review",
7566                                format!(
7567                                    "round {round}: reviewer {} revoted {}",
7568                                    r + 1,
7569                                    rv.vote.label()
7570                                ),
7571                            );
7572                        }
7573                        Err(e) => {
7574                            rec.failed = Some(e.to_string());
7575                            self.state.event(
7576                                "review",
7577                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7578                            );
7579                        }
7580                    }
7581                    reconsideration.push(rec);
7582                }
7583            } else if initial_votes.len() > 1 {
7584                self.state.event(
7585                    "review",
7586                    format!(
7587                        "round {round}: votes agreed ({}) — no reconsideration",
7588                        initial_votes[0].label()
7589                    ),
7590                );
7591            }
7592
7593            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7594            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7595            // A round that already has a blocking finding and a round left to
7596            // try is going back to the fixer no matter what `verify.e2e`
7597            // says, so running it first only spends the loop's slowest step
7598            // (minutes, for a Rust repo's full test suite) on a head about
7599            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7600            // runs once a round has no blocking findings left (see
7601            // `round_is_clean`, which a deferred — empty — `e2e` can never
7602            // satisfy since `blocking` is nonzero whenever this branch is
7603            // taken), and `stop_reviewing` forces a real run before it will
7604            // ever read a deferred round as green.
7605            let defer_e2e =
7606                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7607            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7608                let reason =
7609                    format!("{blocking} blocking finding(s) already required a fix this round");
7610                self.state.event(
7611                    "verify",
7612                    format!(
7613                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7614                         {}); it will run once a round has none left",
7615                        short(&head)
7616                    ),
7617                );
7618                (Vec::new(), false, true, Some(reason))
7619            } else {
7620                let e2e_commands = self.state.config.verify.e2e.clone();
7621                let cache_dir = self.state.config.cache_dir();
7622                let context = format!("round {round}");
7623                let wait = self.repository_wait();
7624                let (e2e, verify_retried) = with_cache_lease(
7625                    &mut self.state,
7626                    cache_dir.as_deref(),
7627                    "e2e",
7628                    "e2e",
7629                    &winner.worktree,
7630                    &head,
7631                    verify_timeout,
7632                    &context,
7633                    &wait,
7634                    |state, budget| {
7635                        let shell = shell.clone();
7636                        let e2e_commands = e2e_commands.clone();
7637                        let worktree = winner.worktree.clone();
7638                        let context = context.clone();
7639                        let winner = winner.clone();
7640                        async move {
7641                            if !e2e_commands.is_empty() {
7642                                // The suite certifies one SHA. Format before it
7643                                // runs, or the commit `pre_gate` makes at the
7644                                // gate owes the whole suite again.
7645                                Self::run_pre_gate(state, &winner).await;
7646                            }
7647                            run_e2e_with_retry(
7648                                state,
7649                                &shell,
7650                                &e2e_commands,
7651                                &worktree,
7652                                budget,
7653                                &context,
7654                            )
7655                            .await
7656                        }
7657                    },
7658                )
7659                .await?;
7660                (e2e, verify_retried, false, None)
7661            };
7662
7663            let expected = records.len();
7664            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7665            let incomplete = answered < expected;
7666            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7667            let policy = self.state.config.graph.incomplete_review;
7668            let clean = round_is_clean(
7669                blocking,
7670                e2e_ok,
7671                answered,
7672                expected,
7673                round_quota_missing,
7674                policy,
7675            );
7676
7677            let mut round_record = ReviewRound {
7678                round,
7679                head: head.clone(),
7680                verified_head: None,
7681                verified_at: None,
7682                reviews: records,
7683                e2e,
7684                verify_retried,
7685                e2e_deferred,
7686                e2e_defer_reason,
7687                fix: None,
7688                blocking,
7689                answered,
7690                expected,
7691                clean,
7692                progressed: false,
7693                vote_split,
7694                reconsideration,
7695                verdict: None,
7696            };
7697            // The final vote per seat is its revote where reconsideration
7698            // ran and answered, its initial vote otherwise — the same
7699            // fallback `tally` uses for a judge whose private vote failed.
7700            round_record.verdict = ReviewVote::worst(
7701                round_record
7702                    .final_votes()
7703                    .into_iter()
7704                    .map(|(_, _, vote)| vote),
7705            );
7706            // Which commit and when magi actually attempted to check —
7707            // known the moment a command was dispatched against `head`,
7708            // whether or not it finished: a resource-blocked attempt still
7709            // targeted a specific commit at a specific time, and leaving
7710            // that unrecorded is exactly what made `verification_summary`
7711            // report a fresh attempt as "commit unknown ... recorded before
7712            // this was tracked", indistinguishable from a genuinely old,
7713            // untracked record. Only a deferred or unconfigured round never
7714            // ran at all and has nothing to record — see
7715            // `ReviewRound::verified_head`'s own doc.
7716            if !matches!(
7717                round_record.e2e_status(),
7718                E2eStatus::Deferred | E2eStatus::NotConfigured
7719            ) {
7720                // HEAD, not the reviewed `head`: `pre_gate` may have committed
7721                // mechanical fixes before the commands ran.
7722                round_record.verified_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
7723                round_record.verified_at = Some(Timestamp::now());
7724            }
7725
7726            if incomplete {
7727                let missing: Vec<String> = round_record
7728                    .reviews
7729                    .iter()
7730                    .filter(|r| r.failed.is_some())
7731                    .map(|r| format!("review-{}", r.reviewer))
7732                    .collect();
7733                self.state.event(
7734                    "review",
7735                    format!(
7736                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7737                        missing.join(", ")
7738                    ),
7739                );
7740            }
7741
7742            if clean {
7743                self.state.event(
7744                    "review",
7745                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7746                        format!(
7747                            "round {round}: clean (warn policy, incomplete panel) — no \
7748                             blocking findings from the seats that answered, verification green"
7749                        )
7750                    } else if incomplete {
7751                        format!(
7752                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7753                             quorum) — no blocking findings from the seats that answered, \
7754                             verification green",
7755                            expected - answered
7756                        )
7757                    } else {
7758                        format!("round {round}: clean — no blocking findings, verification green")
7759                    },
7760                );
7761                self.state.reviews.push(round_record);
7762                self.state.status = RunStatus::Gating;
7763                self.state.save()?;
7764                return Ok(());
7765            }
7766
7767            // Nothing was raised and verification passed, but not every seat
7768            // answered and `round_is_clean` still refused to call it clean —
7769            // either a seat is missing for a reason other than its own quota
7770            // (a crash, a timeout, unparsable output — worth another try), or
7771            // every seat that could have answered lost its quota and nobody
7772            // is left to decide on: re-review rather than send the fixer
7773            // after a round with nothing to fix.
7774            if incomplete && blocking == 0 && e2e_ok {
7775                self.state.reviews.push(round_record);
7776                self.state.save()?;
7777                if round == max_rounds {
7778                    self.state.status = RunStatus::Blocked;
7779                    self.state.event(
7780                        "review",
7781                        format!(
7782                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7783                             refusing to call it clean",
7784                            expected - answered
7785                        ),
7786                    );
7787                    return Ok(());
7788                }
7789                continue;
7790            }
7791
7792            // Nothing for the fixer to act on (`blocking == 0`) and the only
7793            // reason this round is not clean is that magi itself never got
7794            // a command to run — the shared build cache, not the patch (see
7795            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7796            // the fixer would invite a change to appease contention that has
7797            // nothing to do with the diff, and would leave this attempt
7798            // sitting in the next round's prompt as if it were about an
7799            // earlier, superseded commit rather than what it actually is:
7800            // the same head, still waiting to be checked. Wait for it the
7801            // same way the final round's own contention is already handled,
7802            // whatever round this happens to be.
7803            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7804                self.state.reviews.push(round_record);
7805                return self
7806                    .stop_unresolved_review(
7807                        "the round's own verification could not run",
7808                        &shell,
7809                        &winner.worktree,
7810                    )
7811                    .await;
7812            }
7813
7814            if round == max_rounds {
7815                self.state.reviews.push(round_record);
7816                return self
7817                    .stop_reviewing(
7818                        &format!(
7819                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7820                        ),
7821                        &shell,
7822                        &winner.worktree,
7823                    )
7824                    .await;
7825            }
7826
7827            if !self.fix_review_round(&winner, round_record).await? {
7828                return Ok(());
7829            }
7830        }
7831        Ok(())
7832    }
7833
7834    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7835    /// False means admission or continuation policy stopped the review loop.
7836    async fn fix_review_round(
7837        &mut self,
7838        winner: &Candidate,
7839        mut round_record: ReviewRound,
7840    ) -> Result<bool> {
7841        let round = round_record.round;
7842        let shell = self.state.config.shell();
7843        let step = self.run_fix_round(winner, &mut round_record).await?;
7844        if matches!(step, FixStep::Resource) {
7845            self.stop_for_resource()?;
7846            return Ok(false);
7847        }
7848        if self.state.reviews.len() < round {
7849            self.state.reviews.push(round_record);
7850        } else {
7851            self.state.reviews[round - 1] = round_record;
7852        }
7853        self.state.save()?;
7854
7855        // The fixer's own report never came back this round, even after
7856        // `continue_fix_report`'s own budget was spent on it — not an
7857        // ordinary "no report" (dropped stream, quota, plain failure),
7858        // which already reads that way and is left to the existing round
7859        // budget. Stopping here, rather than opening another round, is
7860        // what keeps a next reviewer/fixer wave from ever being
7861        // dispatched onto `winner.worktree` while whatever the seat's
7862        // last call may still have running there is unaccounted for: no
7863        // process liveness check exists (and none is being added — see
7864        // AGENTS.md/this task's own scope), so the only way to honour
7865        // "nothing starts before a valid report returns" is to not start
7866        // anything further on this worktree from this run at all.
7867        if matches!(step, FixStep::Recorded { report_lost: true }) {
7868            self.stop_reviewing(
7869                "the fixer's adoption report never came back, even after resuming its \
7870                         own seat; refusing to start another round against the same worktree \
7871                         while that is unresolved",
7872                &shell,
7873                &winner.worktree,
7874            )
7875            .await?;
7876            return Ok(false);
7877        }
7878
7879        let streak = self
7880            .state
7881            .reviews
7882            .iter()
7883            .rev()
7884            .take_while(|r| !r.progressed)
7885            .count();
7886        if streak >= STAGNANT_LIMIT {
7887            self.stop_reviewing(
7888                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7889                &shell,
7890                &winner.worktree,
7891            )
7892            .await?;
7893            return Ok(false);
7894        }
7895        Ok(true)
7896    }
7897
7898    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7899    async fn run_fix_round(
7900        &mut self,
7901        winner: &Candidate,
7902        round_record: &mut ReviewRound,
7903    ) -> Result<FixStep> {
7904        let round = round_record.round;
7905        let max_rounds = self.state.config.graph.review_rounds;
7906        let run_id = self.state.id.clone();
7907        let prompts = self.state.config.prompts.clone();
7908        let language = self.state.config.graph.language.clone();
7909        let sessions = self.state.config.graph.sessions;
7910        let artifacts = self.merge_repair_artifacts();
7911        let base = self.landing_base();
7912        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7913        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7914        let this_round_verification = round_record.verification_summary(&head);
7915        let all_findings: Vec<_> = round_record
7916            .reviews
7917            .iter()
7918            .flat_map(|r| r.findings.iter().cloned())
7919            .collect();
7920        // Fix. The winner's own implementer seat continues its conversation:
7921        // the competition is over, so context is pure benefit now.
7922        let blocking_findings: Vec<_> = all_findings
7923            .iter()
7924            .filter(|f| f.severity.blocks())
7925            .cloned()
7926            .collect();
7927        let fix_prompt = prompt::fix_in(
7928            &self.state.instruction,
7929            &blocking_findings,
7930            this_round_verification.as_ref(),
7931            round,
7932            max_rounds,
7933            &language,
7934            &self.state.config.graph.github_language,
7935        );
7936        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7937        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7938        let (job, seat, out) = self
7939            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7940                prompt: fix_prompt.clone(),
7941                spec,
7942                seat,
7943                cwd: winner.worktree.clone(),
7944                timeout,
7945                allow_write: true,
7946                sessions,
7947                artifacts: artifacts.clone(),
7948                stem: format!("fix-{round}"),
7949                handover: None,
7950            })
7951            .await;
7952        // Not asked: the round is not recorded (so a resume repeats it),
7953        // no fixer was handed over to, and the seat is left as it was.
7954        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7955            return Ok(FixStep::Resource);
7956        }
7957        let agent_id = seat.agent.clone();
7958
7959        let mut fix = FixRecord {
7960            agent: agent_id,
7961            addressed: Vec::new(),
7962            rejected: Vec::new(),
7963            notes: String::new(),
7964            committed: false,
7965            failed: None,
7966            duration_ms: 0,
7967            continuation: None,
7968        };
7969        let mut continuation = ContinuationRecord::not_needed();
7970        let mut final_seat = seat.clone();
7971        match out {
7972            AgentOutcome::Ok(o) => {
7973                fix.duration_ms = o.duration_ms;
7974                let parsed = verdict::extract_json::<FixReport>(&o.text);
7975                // A parsed report standing next to a command this same
7976                // reply's own CLI never confirmed the exit status of is
7977                // not a resolved answer — the identical `CommandEvidence`
7978                // `state.jobs` renders, read here instead of only on
7979                // display, per the completion judgment and the shown
7980                // record needing to agree.
7981                let incomplete_reason = match &parsed {
7982                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7983                        "the reply parsed, but it reported a command whose own CLI never \
7984                             confirmed an exit status"
7985                            .to_owned(),
7986                    ),
7987                    Ok(_) => None,
7988                    Err(e) => Some(e.to_string()),
7989                };
7990                match incomplete_reason {
7991                    None => {
7992                        let report = parsed.expect("checked Ok above");
7993                        fix.addressed = report.addressed;
7994                        fix.rejected = report.rejected;
7995                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7996                    }
7997                    Some(reason) => {
7998                        let (resumed_seat, resolved, failure, cont) = self
7999                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
8000                            .await;
8001                        fix.duration_ms += cont.cumulative_wait_ms;
8002                        continuation = cont;
8003                        final_seat = resumed_seat;
8004                        match resolved {
8005                            Some(report) => {
8006                                fix.addressed = report.addressed;
8007                                fix.rejected = report.rejected;
8008                                fix.notes =
8009                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
8010                            }
8011                            None => fix.failed = failure,
8012                        }
8013                    }
8014                }
8015            }
8016            // The CLI's raw error JSON is not a fix report to parse.
8017            AgentOutcome::Dropped(o) => {
8018                fix.duration_ms = o.duration_ms;
8019                let why = o
8020                    .dropped
8021                    .as_ref()
8022                    .map(|d| d.why.as_str())
8023                    .unwrap_or("the CLI ended the stream without delivering its answer");
8024                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8025            }
8026            AgentOutcome::Quota(o) => {
8027                self.state.quota.push(QuotaLoss::from_report(
8028                    final_seat.key.clone(),
8029                    "fix",
8030                    o.quota.as_ref(),
8031                ));
8032                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8033            }
8034            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8035        }
8036        if self.state.resource_block.is_some() {
8037            self.state.seats.insert(final_seat.key.clone(), final_seat);
8038            return Ok(FixStep::Resource);
8039        }
8040        fix.continuation = Some(continuation);
8041        self.state.seats.insert(final_seat.key.clone(), final_seat);
8042        let coordination = self.coordinate(&winner.worktree).await?;
8043        if let Ok(r) = git::rescue_commit(
8044            &winner.worktree,
8045            &format!("magi: review round {round} fixes (uncommitted work)"),
8046        )
8047        .await
8048        {
8049            self.state.note_withheld("fix", &r.withheld);
8050        }
8051        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8052        fix.committed = after != before;
8053        // Judged by what `git` says moved against base, never by the
8054        // fixer's own `addressed`/`rejected` count — see
8055        // `ReviewRound::progressed`. Propagated with `?`, the same as the
8056        // `patch` snapshot above: swallowing this error would default
8057        // `diff_after` to empty, which almost always differs from a
8058        // non-empty `patch` and reads as "progressed" — exactly backwards
8059        // for a `git` failure the stagnation check cannot see through.
8060        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8061        drop(coordination);
8062        let progressed = diff_after != patch;
8063        let commit_note = if fix.committed {
8064            "committed"
8065        } else {
8066            "NO new commit"
8067        };
8068        let tree_note = if progressed {
8069            "changed vs base"
8070        } else {
8071            "unchanged vs base"
8072        };
8073        self.state.event(
8074            "fix",
8075            match &fix.failed {
8076                // Distinct on purpose from "0 addressed, 0 rejected": the
8077                // fixer's own diff still landed (blocking counts do keep
8078                // falling round over round), only its adoption report did
8079                // not come back, so this must never read like every
8080                // finding was reviewed and declined.
8081                Some(reason) => {
8082                    format!(
8083                        "round {round}: fixer's adoption report was lost ({reason}); \
8084                             {commit_note}, tree {tree_note}"
8085                    )
8086                }
8087                None => format!(
8088                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8089                         {tree_note}{}",
8090                    fix.addressed.len(),
8091                    fix.rejected.len(),
8092                    if continuation.outcome == ContinuationOutcome::Resumed {
8093                        format!(
8094                            " (adoption report recovered after {} continuation(s))",
8095                            continuation.attempts
8096                        )
8097                    } else {
8098                        String::new()
8099                    },
8100                ),
8101            },
8102        );
8103        round_record.fix = Some(fix);
8104        round_record.progressed = progressed;
8105        Ok(FixStep::Recorded {
8106            report_lost: matches!(
8107                continuation.outcome,
8108                ContinuationOutcome::Exhausted
8109                    | ContinuationOutcome::QuotaLost
8110                    | ContinuationOutcome::NoSession
8111            ),
8112        })
8113    }
8114
8115    /// Decide, from the last recorded round's own verification, whether
8116    /// stopping the review loop is a hand-off or a genuine block.
8117    ///
8118    /// Called once the loop has given up trying — the round budget is spent,
8119    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8120    /// findings still open, never while a round is still clean or the
8121    /// incomplete-panel case handled inline above. Gate and e2e are facts
8122    /// about the tree; a lingering review finding is an opinion, and this
8123    /// workload's own `magi stats` puts reviewer precision low enough
8124    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8125    /// must not by itself stand between a green, verified change and the
8126    /// human who decides what to do with it. A red e2e is not an opinion, so
8127    /// that case still blocks, with the failing command and a tail of its
8128    /// output recorded here rather than left in `run.json` for someone to go
8129    /// find.
8130    ///
8131    /// A round that deferred its own e2e (see [`Config::graph`]'s
8132    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8133    /// only because nothing ran, and treating an empty list as a passing one
8134    /// here is exactly the "deferred painted green" bug this function exists
8135    /// to not have. When the last round's own verification never resolved —
8136    /// deferred on purpose, or a real attempt the shared build cache blocked
8137    /// — this makes (or retries) the real run, on the actual worktree this
8138    /// loop is about to stop touching, before deciding anything. A
8139    /// resource-blocked attempt is likewise never read as either green or
8140    /// red: it is evidence about the machine, not the patch (see
8141    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8142    /// blocked cache leaves this call without deciding rather than guessing
8143    /// — the caller retries on a later reentry.
8144    /// Record, once, that the review loop handed off over a blocking finding
8145    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8146    /// `land` asks the owner even with `land_approval` off. Called from every
8147    /// path that concludes `Gating`; a reentry keeps the first record.
8148    fn record_contested_handoff(&mut self) {
8149        if self.state.contested_handoff.is_some() {
8150            return;
8151        }
8152        let Some(contested) = self
8153            .state
8154            .reviews
8155            .last()
8156            .and_then(ReviewRound::contested_handoff)
8157        else {
8158            return;
8159        };
8160        self.state.event(
8161            "review",
8162            format!(
8163                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8164                 wait for the owner's approval",
8165                contested.findings.len(),
8166                contested.rejecters.len()
8167            ),
8168        );
8169        self.state.contested_handoff = Some(contested);
8170    }
8171
8172    async fn stop_unresolved_review(
8173        &mut self,
8174        why: &str,
8175        shell: &[String],
8176        worktree: &Path,
8177    ) -> Result<()> {
8178        self.stop_reviewing_inner(why, shell, worktree, true).await
8179    }
8180
8181    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8182        self.stop_reviewing_inner(why, shell, worktree, false).await
8183    }
8184
8185    async fn stop_reviewing_inner(
8186        &mut self,
8187        why: &str,
8188        shell: &[String],
8189        worktree: &Path,
8190        repairable: bool,
8191    ) -> Result<()> {
8192        let round_idx = self.state.reviews.len() - 1;
8193        // A deferred round and a resource-blocked one are the same shape
8194        // here: neither has a real result yet, and both get one more
8195        // attempt. Read off `e2e_status` — the single source for this —
8196        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8197        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8198        // still retries instead of being read as a settled result the
8199        // instant it stops being empty.
8200        let needs_catchup_run = matches!(
8201            self.state.reviews[round_idx].e2e_status(),
8202            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8203        );
8204        if needs_catchup_run {
8205            let round = self.state.reviews[round_idx].round;
8206            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8207            let commands = self.state.config.verify.e2e.clone();
8208            let before_head = git::rev_parse(worktree, "HEAD").await?;
8209            let winner = self.state.winner().cloned();
8210            let cache_dir = self.state.config.cache_dir();
8211            let context = format!(
8212                "round {round}: verification unresolved, catching up before the final decision"
8213            );
8214            let wait = self.repository_wait();
8215            let (outcomes, verify_retried) = with_cache_lease(
8216                &mut self.state,
8217                cache_dir.as_deref(),
8218                "e2e",
8219                "e2e",
8220                worktree,
8221                &before_head,
8222                timeout,
8223                &context,
8224                &wait,
8225                |state, budget| {
8226                    let shell = shell.to_vec();
8227                    let commands = commands.clone();
8228                    let context = context.clone();
8229                    async move {
8230                        if let Some(winner) = winner.filter(|_| !commands.is_empty()) {
8231                            // Same reason as in the round itself: format before
8232                            // the suite, not after it.
8233                            Self::run_pre_gate(state, &winner).await;
8234                        }
8235                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8236                            .await
8237                    }
8238                },
8239            )
8240            .await?;
8241            // After `pre_gate`: the commit the commands actually ran against.
8242            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8243            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8244                self.state.record_base_progress(&attempted_head);
8245            }
8246            let last = &mut self.state.reviews[round_idx];
8247            last.e2e = outcomes;
8248            last.verify_retried = verify_retried;
8249            // Always the commit and time this attempt actually targeted,
8250            // whether or not it happens to equal the reviewed `head` and
8251            // whether or not a command finished — see
8252            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8253            // attempt is recorded too, so a later reader sees "attempted
8254            // again at T2" rather than silence.
8255            last.verified_head = Some(attempted_head);
8256            last.verified_at = Some(Timestamp::now());
8257            if verify_inconclusive(&last.e2e) {
8258                // Still not a real result: `e2e_deferred` is left exactly
8259                // as it was, so `needs_catchup_run` above reads
8260                // `ResourceBlocked` (via `e2e_status`, which checks
8261                // `resource_blocked` before `e2e_deferred`) and retries
8262                // again on the next reentry, rather than recording
8263                // contention as a red e2e and blocking the run on it.
8264                self.state.save()?;
8265                return Ok(());
8266            }
8267            last.e2e_deferred = false;
8268            // A complete panel may have been saved as unclean solely because
8269            // verification could not run. Refresh that verdict with the new
8270            // outcomes before a granted repair consumes the round's result.
8271            // Incomplete panels retain their original policy/quota decision.
8272            if last.answered == last.expected {
8273                last.clean = round_is_clean(
8274                    last.blocking,
8275                    last.e2e.iter().all(CommandOutcome::ok),
8276                    last.answered,
8277                    last.expected,
8278                    0,
8279                    self.state.config.graph.incomplete_review,
8280                );
8281            }
8282        }
8283        let last = &self.state.reviews[round_idx];
8284        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8285
8286        match last.e2e_status() {
8287            E2eStatus::Failed => {
8288                let red: Vec<String> = last
8289                    .e2e
8290                    .iter()
8291                    .filter(|o| !o.ok())
8292                    .map(|o| {
8293                        format!(
8294                            "`{}` -> {:?}\n{}",
8295                            o.command,
8296                            o.code,
8297                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8298                        )
8299                    })
8300                    .collect();
8301                let can_repair = last.blocking == 0 && last.fix.is_none();
8302                self.state.reviews[round_idx].clean = false;
8303                self.state
8304                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8305                self.state.status = RunStatus::Blocked;
8306                if repairable
8307                    && can_repair
8308                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8309                {
8310                    self.state.status = RunStatus::Reviewing;
8311                }
8312            }
8313            // `needs_catchup_run` above already retried once this call; if
8314            // it is still blocked, this is magi's own admission it could
8315            // not get a command to run, never a verdict on the patch — the
8316            // run is left exactly where a later reentry can retry again.
8317            E2eStatus::ResourceBlocked => {
8318                self.state.event(
8319                    "review",
8320                    format!(
8321                        "{why}; e2e could not run (resource unavailable); not \
8322                         deciding yet"
8323                    ),
8324                );
8325            }
8326            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8327                self.state.event(
8328                    "review",
8329                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8330                );
8331                self.record_contested_handoff();
8332                self.state.status = RunStatus::Gating;
8333            }
8334        }
8335        self.state.save()?;
8336        Ok(())
8337    }
8338
8339    // ----------------------------------------------------------------- gate
8340
8341    async fn gate(&mut self) -> Result<()> {
8342        if self.hold_base_sync().await? {
8343            return Ok(());
8344        }
8345        // Judged by the review record itself, not by `status`: a solo
8346        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8347        // reentry (see `judge`), and trusting it here is exactly how a run
8348        // that exhausted its review budget got gated and merged a second
8349        // time around. `review_conclusion` recomputes the review loop's own
8350        // verdict from the round records themselves — `Gating` for a clean
8351        // round or a hand-off (see `stop_reviewing`), anything else means the
8352        // loop is still going or genuinely blocked.
8353        // A base the winner could not be replayed onto is a decision, not a
8354        // round: there is no landing tree to gate. Read as its own record for
8355        // the same reason the review verdict is.
8356        if self.state.status == RunStatus::Failed
8357            || self
8358                .state
8359                .base_sync
8360                .as_ref()
8361                .is_some_and(|s| s.conflict.is_some())
8362            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8363                != Some(RunStatus::Gating)
8364        {
8365            return Ok(());
8366        }
8367        let Some(winner) = self.state.winner().cloned() else {
8368            return Ok(());
8369        };
8370        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8371        self.state.reconcile_verification_head(&head);
8372        self.state.save()?;
8373        let wait = self.repository_wait();
8374        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8375            return Ok(());
8376        }
8377        if self.state.gate_ran {
8378            // `review_loop` derives its conclusion from the clean review
8379            // record on every reentry and therefore puts a completed run back
8380            // in `Gating`. A recorded gate is a stronger, terminal fact:
8381            // retain its original command output (or lack of any, for a repo
8382            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8383            // doc) and restore `Blocked` on a real failure rather than
8384            // pretending the command is still running or running it a second
8385            // time. `gate_ran == false` remains the only shape — unattempted,
8386            // or a resource-blocked retry — that may still need to execute a
8387            // command.
8388            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8389                self.state.status = RunStatus::Blocked;
8390                self.state.save()?;
8391            }
8392            return Ok(());
8393        }
8394        self.state.status = RunStatus::Gating;
8395        let mut outcomes = self.run_gate(&winner).await?;
8396        let mut attempted_head = self.state.gate_head.clone();
8397        loop {
8398            // A resource-blocked outcome means the gate command never actually
8399            // ran - the shared build cache could not be acquired or confirmed
8400            // fresh in time - which is evidence about the machine, not about
8401            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8402            // Recording it as a red gate would mark a run `Blocked` on nothing
8403            // but contention magi has already logged; leaving `self.state.gate`
8404            // empty and `self.state.gate_ran` false instead keeps the shape
8405            // this function already treats as "still needs to run" (see the
8406            // early-return above), so the next call retries the command
8407            // rather than concluding anything.
8408            if verify_inconclusive(&outcomes) {
8409                self.state.save()?;
8410                return Ok(());
8411            }
8412            if outcomes.iter().all(CommandOutcome::ok) {
8413                break;
8414            }
8415            let fixes_before = self.state.gate_fixes.len();
8416            match self.gate_fix_round(&winner, &outcomes).await? {
8417                GateFix::Retry => {
8418                    outcomes = self.run_gate(&winner).await?;
8419                    attempted_head = self.state.gate_head.clone();
8420                }
8421                GateFix::Stop => {
8422                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8423                    if self.state.gate_fixes.len() > fixes_before
8424                        && attempted_head.as_deref() != Some(current.as_str())
8425                        && verify_current_head(&mut self.state, &winner, &wait).await?
8426                    {
8427                        // A new SHA owes a gate even when a spent fixer changed only
8428                        // commit metadata. This does not grant another repair round.
8429                        outcomes = self.run_gate(&winner).await?;
8430                        attempted_head = self.state.gate_head.clone();
8431                    } else {
8432                        break;
8433                    }
8434                }
8435                GateFix::Defer => {
8436                    self.state.save()?;
8437                    return Ok(());
8438                }
8439            }
8440        }
8441        let passed = outcomes.iter().all(CommandOutcome::ok);
8442        self.state.gate = outcomes;
8443        self.state.gate_ran = true;
8444        self.state.gate_head = attempted_head;
8445        if !self.state.gate.is_empty()
8446            && !verify_inconclusive(&self.state.gate)
8447            && let Some(head) = self.state.gate_head.clone()
8448        {
8449            self.state.record_base_progress(&head);
8450        }
8451        self.state.save()?;
8452        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8453            return Ok(());
8454        }
8455        if !passed {
8456            self.state.status = RunStatus::Blocked;
8457            let spent = self.state.gate_fixes.len();
8458            self.state.event(
8459                "gate",
8460                if spent == 0 {
8461                    "gate failed; not merging".to_owned()
8462                } else {
8463                    format!("gate failed after {spent} gate-fix round(s); not merging")
8464                },
8465            );
8466        }
8467        self.state.save()?;
8468        Ok(())
8469    }
8470
8471    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8472    /// changed into one commit. Runs before `verify.e2e` in the review loop
8473    /// and again before the gate commands, always on the winner and never on
8474    /// a candidate awaiting judging.
8475    ///
8476    /// Before e2e as well as before the gate because command evidence belongs
8477    /// to a SHA ([`verify_current_head`]): a commit made only at the gate is a
8478    /// commit e2e never saw, and the suite ran again for a formatter's diff -
8479    /// 5 of the 22 gated runs on this repository in November 2026. The gate's
8480    /// own call then finds nothing left to change. The commands must be
8481    /// idempotent; one that is not makes a commit per call.
8482    ///
8483    /// Never fails the run: a non-zero exit or timeout is a warning and a
8484    /// recorded outcome, and the gate remains the single arbiter. Nothing
8485    /// configured means nothing happens - no event, no commit. `commit_all`
8486    /// commits any leftover change under the neutral identity and returns
8487    /// `false` when the tree is clean, so no empty commit is ever made.
8488    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8489        let commands = state.config.verify.pre_gate.clone();
8490        if commands.is_empty() {
8491            return;
8492        }
8493        let shell = state.config.shell();
8494        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8495        let (outcomes, _) = run_commands(
8496            state,
8497            "pre_gate",
8498            "pre_gate",
8499            0,
8500            &shell,
8501            &commands,
8502            &winner.worktree,
8503            timeout,
8504        )
8505        .await;
8506        for o in &outcomes {
8507            if !o.ok() {
8508                tracing::warn!(
8509                    "pre_gate `{}` failed ({:?}); the gate decides",
8510                    o.command,
8511                    o.code
8512                );
8513            }
8514            state.event(
8515                "pre_gate",
8516                format!(
8517                    "`{}` -> {}",
8518                    o.command,
8519                    if o.ok() {
8520                        "pass".to_owned()
8521                    } else {
8522                        format!(
8523                            "FAIL ({:?})\n{}",
8524                            o.code,
8525                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8526                        )
8527                    }
8528                ),
8529            );
8530        }
8531        state.pre_gate = outcomes;
8532        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8533            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8534                Ok(head) => {
8535                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8536                    state.pre_gate_commit = Some(head);
8537                }
8538                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8539            },
8540            Ok(false) => {}
8541            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8542        }
8543        if let Err(e) = state.save() {
8544            tracing::warn!("could not persist the pre_gate record: {e:#}");
8545        }
8546    }
8547
8548    /// Run `verify.gate` once against the winner's current tree, logging one
8549    /// event per command. Empty when nothing is configured.
8550    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8551        let wait = self.repository_wait();
8552        run_landing_gate(&mut self.state, winner, &wait).await
8553    }
8554
8555    /// One bounded fix round for a failing gate.
8556    ///
8557    /// The fixer is told the failure came from the gate itself, not from a
8558    /// reviewer, and is shown the failed commands, their exit codes and a tail
8559    /// of their output - whatever `[verify].gate` holds, nothing here knows
8560    /// what those commands run. Only a normal non-zero exit that printed
8561    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8562    /// command or a full disk says nothing about the code, and a fixer sent
8563    /// after it can only appease the machine. The round is judged by what git
8564    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8565    /// again before the gate does, so a fix cannot trade a green gate for a
8566    /// red e2e unnoticed.
8567    async fn gate_fix_round(
8568        &mut self,
8569        winner: &Candidate,
8570        outcomes: &[CommandOutcome],
8571    ) -> Result<GateFix> {
8572        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8573            g.phase == crate::repair_budget::Phase::Fixing
8574                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8575        }) {
8576            self.state.gate_fixes.len().saturating_add(1)
8577        } else {
8578            self.state.config.graph.gate_fix_rounds
8579        };
8580        let spent = self.state.gate_fixes.len();
8581        if spent >= cap {
8582            if cap > 0 {
8583                self.state.event(
8584                    "gate",
8585                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8586                );
8587            }
8588            return Ok(GateFix::Stop);
8589        }
8590        if !gate_fixable(outcomes) {
8591            self.state.event(
8592                "gate",
8593                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8594                 command or similar); not spending a fix round on it",
8595            );
8596            return Ok(GateFix::Stop);
8597        }
8598        let min_free = self.state.config.disk.min_free_bytes;
8599        if min_free > 0 {
8600            match crate::disk::free_bytes(&winner.worktree) {
8601                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8602                Ok(free) => {
8603                    self.state.event(
8604                        "gate",
8605                        format!(
8606                            "only {free} bytes free ({min_free} required by `[disk] \
8607                             min_free_bytes`); not spending a fix round on a failure the disk \
8608                             may explain"
8609                        ),
8610                    );
8611                    return Ok(GateFix::Stop);
8612                }
8613                Err(e) => {
8614                    self.state.event(
8615                        "gate",
8616                        format!("free disk space could not be measured ({e:#}); no fix round"),
8617                    );
8618                    return Ok(GateFix::Stop);
8619                }
8620            }
8621        }
8622
8623        let attempt = spent + 1;
8624        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8625        let base = self.landing_base();
8626        let fix_prompt = prompt::gate_fix_in(
8627            &self.state.instruction,
8628            &failed,
8629            attempt,
8630            cap,
8631            &self.state.config.graph.language,
8632            &self.state.config.graph.github_language,
8633        );
8634        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8635        let sessions = self.state.config.graph.sessions;
8636        let artifacts = self.merge_repair_artifacts();
8637        self.state.event(
8638            "gate",
8639            format!("gate failed; gate-fix round {attempt} of {cap}"),
8640        );
8641        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8642        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8643        // Persist before invoking the fixer: it may commit and then be
8644        // interrupted before its result (or the following e2e) is saved.
8645        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8646            head: before.clone(),
8647            passed: false,
8648            outcomes: Vec::new(),
8649        });
8650        self.state.save()?;
8651        let (_, seat, out) = self
8652            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8653                prompt: fix_prompt.clone(),
8654                spec,
8655                seat,
8656                cwd: winner.worktree.clone(),
8657                timeout,
8658                allow_write: true,
8659                sessions,
8660                artifacts: artifacts.clone(),
8661                stem: format!("gate-fix-{attempt}"),
8662                handover: None,
8663            })
8664            .await;
8665        // The fixer was never asked: no round is spent (nothing is recorded)
8666        // and a later reentry asks the same agent again.
8667        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8668            return Ok(GateFix::Defer);
8669        }
8670        let mut record = GateFixRecord {
8671            agent: seat.agent.clone(),
8672            failed,
8673            notes: String::new(),
8674            committed: false,
8675            error: None,
8676        };
8677        match out {
8678            AgentOutcome::Ok(o) => {
8679                // A missing report is not a failed fix: the round is judged
8680                // by the tree below, and the report only carries prose.
8681                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8682                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8683                }
8684            }
8685            AgentOutcome::Dropped(_) => {
8686                record.error = Some("the CLI dropped the stream".to_owned());
8687            }
8688            AgentOutcome::Quota(o) => {
8689                self.state.quota.push(QuotaLoss::from_report(
8690                    seat.key.clone(),
8691                    "gate-fix",
8692                    o.quota.as_ref(),
8693                ));
8694                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8695            }
8696            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8697        }
8698        self.state.seats.insert(seat.key.clone(), seat);
8699        let coordination = self.coordinate(&winner.worktree).await?;
8700        if let Ok(r) = git::rescue_commit(
8701            &winner.worktree,
8702            &format!("magi: gate fix {attempt} (uncommitted work)"),
8703        )
8704        .await
8705        {
8706            self.state.note_withheld("gate-fix", &r.withheld);
8707        }
8708        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8709        record.committed = after != before;
8710        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8711        let note = record.error.clone();
8712        self.state.gate_fixes.push(record);
8713        self.state.save()?;
8714        if !changed {
8715            self.state.event(
8716                "gate",
8717                match note {
8718                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8719                    None => format!("gate-fix round {attempt}: the tree did not change"),
8720                },
8721            );
8722            return Ok(GateFix::Stop);
8723        }
8724        self.state.event(
8725            "gate",
8726            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8727        );
8728
8729        drop(coordination);
8730        let wait = self.repository_wait();
8731        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8732    }
8733
8734    // ---------------------------------------------------------------- merge
8735
8736    /// One read-only rewrite by the summary's author; a text the gate still
8737    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8738    /// applies on their word or on silence. `None` means the run is parked on
8739    /// that question and nothing may be pushed or opened yet.
8740    async fn guarded_pr_message(
8741        &mut self,
8742        winner: &Candidate,
8743        facts: Option<&BranchFacts>,
8744        posting: bool,
8745    ) -> Result<Option<PrMessage>> {
8746        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8747        let head = match final_head.as_deref() {
8748            Some(h) => crate::pr_outcome::FinalHead::At(h),
8749            None => crate::pr_outcome::FinalHead::Unknown,
8750        };
8751        // Only a pull request that is about to be posted pays for a summary.
8752        let summarized = if posting {
8753            Some(self.summarize_change(winner, final_head.as_deref()).await)
8754        } else {
8755            None
8756        };
8757        // A busy shared resource is not a permanent cause: publishing an
8758        // "unavailable" description would stick (a later head move never
8759        // edits the body). Stop before anything is pushed or opened; the
8760        // run is `Blocked` and a resume asks the summary again.
8761        if self.state.resource_block.is_some() {
8762            self.state.event(
8763                "pr-summary",
8764                "a shared resource was busy; nothing was published, resume to summarize again",
8765            );
8766            return Ok(None);
8767        }
8768        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8769            anyhow::bail!(
8770                "the final commit moved during public description generation; retry before publishing"
8771            );
8772        }
8773        let note = match (&summarized, &self.state.change_summary) {
8774            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8775            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8776            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8777            (None, _) => ChangeNote::NotAsked,
8778        };
8779        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8780        if !posting {
8781            let identity = crate::scrub::Identity::current();
8782            return Ok(Some(PrMessage {
8783                title: crate::scrub::scrub(&pr.title, &identity),
8784                body: crate::scrub::scrub(&pr.body, &identity),
8785            }));
8786        }
8787        // This very text was already rejected and asked about: no second
8788        // judge call, rewrite or question, just carry the record on.
8789        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8790        if let Some(known) = self
8791            .state
8792            .github_text
8793            .clone()
8794            .filter(|g| g.fingerprint == raw_fp)
8795        {
8796            return self.settle_withheld(winner, &pr, known).await;
8797        }
8798        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8799        let mut violations =
8800            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8801        let identity = crate::scrub::Identity::current();
8802        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8803            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8804            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8805        {
8806            violations.push(crate::github_text::Violation::SensitiveData);
8807        }
8808        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8809        // only a language violation starts one; sensitive data alone goes
8810        // straight to the owner, with the redacted text as the fallback.
8811        let has_language = violations
8812            .iter()
8813            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8814        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8815            self.state.event(
8816                "github-text",
8817                if has_language {
8818                    format!("description rejected: {violations:?}; requesting one rewrite")
8819                } else {
8820                    format!("description rejected: {violations:?}; asking the owner")
8821                },
8822            );
8823            let mut rewritten = false;
8824            let mut rejected_rewrite = None;
8825            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8826            {
8827                let output = text;
8828                if let Some(title) = summary_title(&output) {
8829                    // Judge the redacted candidate that will actually be posted.
8830                    // Sensitive data alone must not discard a useful rewrite.
8831                    let raw_body = summary_without_title(&output);
8832                    let clean_title = crate::scrub::scrub(&title, &identity);
8833                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8834                    let redacted = clean_title != title || body != raw_body;
8835                    let title = clean_title;
8836                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8837                    if !complete {
8838                        self.state.event(
8839                            "github-text",
8840                            "description rewrite omitted required findings",
8841                        );
8842                    }
8843                    if !body.trim().is_empty() {
8844                        body.push_str("\n\n");
8845                        body.push_str(&public_pr_evidence(&self.state, head));
8846                    }
8847                    let marker = format!("magi:run/{}", self.state.id);
8848                    if !body.trim().is_empty() && !body.contains(&marker) {
8849                        body.push_str(&format!("\n\n{marker}"));
8850                    }
8851                    let rewrite = PrMessage {
8852                        title: title.clone(),
8853                        body: body.clone(),
8854                    };
8855                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8856                    let mut failed = crate::github_text::check_for(
8857                        &self.state.config.graph,
8858                        &title,
8859                        &body,
8860                        redecision,
8861                    );
8862                    if (!crate::github_text::shareable(&title)
8863                        || !crate::github_text::shareable(&body))
8864                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8865                    {
8866                        failed.push(crate::github_text::Violation::SensitiveData);
8867                    }
8868                    if complete
8869                        && !title.trim().is_empty()
8870                        && !body.trim().is_empty()
8871                        && !failed.is_empty()
8872                    {
8873                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8874                            title: title.clone(),
8875                            body: body.clone(),
8876                            failed: failed
8877                                .iter()
8878                                .map(|v| crate::github_text::category(*v).to_owned())
8879                                .collect(),
8880                        });
8881                    }
8882                    if complete
8883                        && !title.trim().is_empty()
8884                        && !raw_body.trim().is_empty()
8885                        && failed.is_empty()
8886                    {
8887                        decision = redecision;
8888                        pr = PrMessage { title, body };
8889                        rewritten = true;
8890                        self.state
8891                            .event("github-text", "description rewrite passed");
8892                        if redacted {
8893                            self.state.event(
8894                                "github-text",
8895                                "sensitive data removed from description rewrite",
8896                            );
8897                        }
8898                    }
8899                }
8900            }
8901            if !rewritten {
8902                self.state.event(
8903                    "github-text",
8904                    "description rewrite unavailable or rejected; asking the owner",
8905                );
8906                if let Some(w) =
8907                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8908                {
8909                    // A different text than the one asked about before: that
8910                    // question no longer describes anything.
8911                    self.retire_github_text_question("the withheld text changed");
8912                    let record = crate::run::GithubTextAsk {
8913                        rewrite: rejected_rewrite,
8914                        fingerprint: raw_fp,
8915                        title: w.title,
8916                        body: w.body,
8917                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8918                        question: None,
8919                        asks: 0,
8920                        resolved: false,
8921                        chosen_title: None,
8922                        choice: None,
8923                        publication: None,
8924                    };
8925                    return self.settle_withheld(winner, &pr, record).await;
8926                }
8927            }
8928        }
8929        let (title, body) =
8930            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8931        Ok(Some(PrMessage { title, body }))
8932    }
8933
8934    /// Describe the whole change from its final diff in either run mode.
8935    ///
8936    /// The summary is tied to the final commit and to the merge base of the
8937    /// resolved remote base; a record for any other pair is dropped and the
8938    /// agent asked again, so a description never outlives the diff it was
8939    /// written from. The seat is read-only and the prompt tells it to read the
8940    /// diff and the code around it. `[roles] synthesizer` selects the agents
8941    /// (unset: the default pick), each asked at most once, advancing on an
8942    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8943    /// carries the reason it is unavailable; nothing is invented then.
8944    async fn summarize_change(
8945        &mut self,
8946        winner: &Candidate,
8947        head: Option<&str>,
8948    ) -> std::result::Result<(), String> {
8949        let Some(head) = head else {
8950            return Err("the final commit could not be read".to_owned());
8951        };
8952        let remote = self.state.config.merge.remote.clone();
8953        let tracking = format!("{remote}/{}", self.state.base_branch);
8954        if !git::rev_exists(&winner.worktree, &tracking).await {
8955            return Err(format!("the remote base `{tracking}` could not be read"));
8956        }
8957        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8958            Ok(b) if !b.is_empty() => b,
8959            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8960        };
8961        let evidence = crate::pr_outcome::summary_evidence(
8962            &self.state,
8963            crate::pr_outcome::FinalHead::At(head),
8964        );
8965        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8966        if self
8967            .state
8968            .change_summary
8969            .as_ref()
8970            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8971        {
8972            return Ok(());
8973        }
8974        if self.state.change_summary.take().is_some() {
8975            self.state.event(
8976                "pr-summary",
8977                "the final diff or evidence changed; the earlier change summary was discarded",
8978            );
8979        }
8980        // Save the source snapshot even if no generation agent is available.
8981        if let Err(e) = write_artifact(
8982            &self.state,
8983            &format!("public-description-{fingerprint}.md"),
8984            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8985        ) {
8986            tracing::warn!("could not save public description evidence: {e:#}");
8987        }
8988        let chain = match agent::pick_chain(
8989            &self.state.config.agents,
8990            self.state.config.roles.synthesizer.as_ref(),
8991            &agent::installed,
8992            "synthesizer",
8993        ) {
8994            Ok(chain) => chain,
8995            Err(e) => {
8996                tracing::warn!("no agent for the change summary: {e:#}");
8997                self.state.event("pr-summary", "no summary agent available");
8998                return Err("no agent was available to summarize it".to_owned());
8999            }
9000        };
9001        let subjects = git::subjects(&winner.worktree, &base, head)
9002            .await
9003            .unwrap_or_default();
9004        let stat = git::diff_stat(&winner.worktree, &base, head)
9005            .await
9006            .unwrap_or_default();
9007        let stat: String = stat.chars().take(6_000).collect();
9008        let mut prompt = crate::prompt::change_summary_in(
9009            &base,
9010            head,
9011            &subjects,
9012            stat.trim(),
9013            &self.state.config.graph.github_language,
9014        );
9015        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
9016        let sessions = self.state.config.graph.sessions;
9017        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
9018        let prompts = self.state.config.prompts.clone();
9019        let cache = self.state.config.cache_dir();
9020        let run = self.state.id.clone();
9021        for (i, spec) in chain.into_iter().enumerate() {
9022            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
9023            let github_spec = match agent::for_github_text(
9024                &spec,
9025                &self.state.config.graph.github_language,
9026                &winner.worktree,
9027            ) {
9028                Ok(spec) => spec,
9029                Err(e) => {
9030                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9031                    continue;
9032                }
9033            };
9034            let job = SeatJob {
9035                spec: github_spec,
9036                seat,
9037                cwd: winner.worktree.clone(),
9038                prompt: prompt.clone(),
9039                timeout,
9040                allow_write: false,
9041                sessions,
9042                artifacts: agent::artifacts_dir(&self.state.dir()),
9043                stem: if i == 0 {
9044                    "change-summary".to_owned()
9045                } else {
9046                    format!("change-summary-{}", spec.id)
9047                },
9048                handover: None,
9049            };
9050            let ctx = WaveCtx {
9051                run: &run,
9052                node: "pr-summary",
9053                prompts: &prompts,
9054                cache: cache.as_deref(),
9055                round: None,
9056                carry_seats: false,
9057                pauses: Some((&self.pause, &self.interrupt)),
9058            };
9059            let (seat, outcome) =
9060                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9061            self.state.seats.insert(seat.key.clone(), seat);
9062            let text = match outcome {
9063                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9064                AgentOutcome::ResourceBlocked(why) => {
9065                    // No agent was asked: the run stops resumably before
9066                    // anything is published (see `guarded_pr_message`).
9067                    if self.state.resource_block.is_none() {
9068                        self.state.resource_block = Some(crate::run::ResourceBlock {
9069                            cause: crate::run::ResourceBlockCause::Other,
9070                            node: "pr-summary".to_owned(),
9071                            kind: "shared resource".to_owned(),
9072                            detail: why,
9073                            at: Timestamp::now(),
9074                        });
9075                    }
9076                    return Err("a shared resource was busy".to_owned());
9077                }
9078                _ => continue,
9079            };
9080            let blind = &self.state.config.blind;
9081            let Some(title) = summary_title(&text) else {
9082                continue;
9083            };
9084            let body: String = summary_without_title(&text)
9085                .lines()
9086                .filter(|l| !l.contains("magi:run/"))
9087                .collect::<Vec<_>>()
9088                .join("\n");
9089            let body = blind::sanitize_prose(body.trim(), blind);
9090            let title = blind::sanitize_prose(&title, blind);
9091            if body.trim().is_empty() || title.trim().is_empty() {
9092                continue;
9093            }
9094            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9095                continue;
9096            }
9097            // The branch must still be the one the agent was shown.
9098            if git::rev_parse(&winner.worktree, "HEAD")
9099                .await
9100                .ok()
9101                .as_deref()
9102                != Some(head)
9103            {
9104                return Err("the branch moved while the summary was written".to_owned());
9105            }
9106            if git::merge_base(&winner.worktree, &tracking, head)
9107                .await
9108                .ok()
9109                .as_deref()
9110                != Some(base.as_str())
9111                || crate::pr_outcome::summary_evidence(
9112                    &self.state,
9113                    crate::pr_outcome::FinalHead::At(head),
9114                ) != evidence
9115            {
9116                return Err("the summary evidence changed while it was written".to_owned());
9117            }
9118            self.state.change_summary = Some(crate::run::ChangeSummary {
9119                head: head.to_owned(),
9120                base,
9121                evidence: fingerprint,
9122                title,
9123                body,
9124            });
9125            self.state.event(
9126                "pr-summary",
9127                format!("change summary written by {} for the final diff", spec.id),
9128            );
9129            if let Err(e) = self.state.save() {
9130                tracing::warn!("could not save the change summary: {e:#}");
9131            }
9132            return Ok(());
9133        }
9134        self.state
9135            .event("pr-summary", "no agent produced a change summary");
9136        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9137    }
9138
9139    /// One read-only rewrite request for a withheld description. The
9140    /// implementer's own conversation is tried first when it exists and can be
9141    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9142    /// writes it in a separate conversation, skipping the implementer's agent.
9143    /// The prompt carries the whole text, so no agent needs any memory. Each
9144    /// agent is asked at most once and the chain advances on an error, quota,
9145    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9146    /// that has run out of quota must not end the rewrite. Nothing is recorded
9147    /// as a handover.
9148    async fn rewrite_pr_text(
9149        &mut self,
9150        winner: &Candidate,
9151        pr: &PrMessage,
9152        violations: &[crate::github_text::Violation],
9153    ) -> Option<String> {
9154        let prompt = format!(
9155            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9156            pr.title, pr.body
9157        );
9158        let sessions = self.state.config.graph.sessions;
9159        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9160        // (spec, the resumed implementer's seat). A fallback seat is created
9161        // only when its turn comes, so an unused fallback leaves no record.
9162        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9163        let key = format!("impl-{}", winner.label);
9164        if let Some(spec) = self
9165            .state
9166            .config
9167            .agents
9168            .iter()
9169            .find(|a| a.id == winner.agent)
9170            .cloned()
9171            && let Some(existing) = self.state.seats.get(&key)
9172            && existing.agent == spec.id
9173            && agent::has_session(spec.kind, existing, sessions)
9174        {
9175            attempts.push((spec, Some(existing.clone())));
9176        }
9177        match agent::pick_chain(
9178            &self.state.config.agents,
9179            self.state.config.roles.language_judge.as_ref(),
9180            &agent::installed,
9181            "language_judge",
9182        ) {
9183            Ok(chain) => {
9184                for spec in chain {
9185                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9186                        continue;
9187                    }
9188                    attempts.push((spec, None));
9189                }
9190            }
9191            Err(e) if attempts.is_empty() => {
9192                tracing::warn!("no agent for the description rewrite: {e:#}");
9193                self.state
9194                    .event("github-text", "no rewrite agent available");
9195                return None;
9196            }
9197            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9198        }
9199        let prompts = self.state.config.prompts.clone();
9200        let cache = self.state.config.cache_dir();
9201        let run = self.state.id.clone();
9202        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9203            let resumed = resumed_seat.is_some();
9204            let seat = resumed_seat.unwrap_or_else(|| {
9205                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9206            });
9207            let github_spec = match agent::for_github_text(
9208                &spec,
9209                &self.state.config.graph.github_language,
9210                &winner.worktree,
9211            ) {
9212                Ok(spec) => spec,
9213                Err(e) => {
9214                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9215                    continue;
9216                }
9217            };
9218            let job = SeatJob {
9219                spec: github_spec,
9220                seat,
9221                cwd: winner.worktree.clone(),
9222                prompt: prompt.clone(),
9223                timeout: retry_budget(timeout_full, resumed),
9224                allow_write: false,
9225                sessions,
9226                artifacts: agent::artifacts_dir(&self.state.dir()),
9227                stem: if i == 0 {
9228                    "github-text-rewrite".to_owned()
9229                } else {
9230                    format!("github-text-rewrite-{}", spec.id)
9231                },
9232                handover: None,
9233            };
9234            let ctx = WaveCtx {
9235                run: &run,
9236                node: "github-text",
9237                prompts: &prompts,
9238                cache: cache.as_deref(),
9239                round: None,
9240                carry_seats: false,
9241                pauses: Some((&self.pause, &self.interrupt)),
9242            };
9243            let (seat, outcome) =
9244                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9245            self.state.seats.insert(seat.key.clone(), seat);
9246            match outcome {
9247                AgentOutcome::Ok(o)
9248                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9249                {
9250                    return Some(o.text);
9251                }
9252                AgentOutcome::ResourceBlocked(_) => return None,
9253                _ => {}
9254            }
9255        }
9256        None
9257    }
9258
9259    /// Abandon the standing posting-gate question, if any.
9260    fn retire_github_text_question(&mut self, why: &str) {
9261        let Some(id) = self
9262            .state
9263            .github_text
9264            .as_ref()
9265            .and_then(|g| g.question.clone())
9266        else {
9267            return;
9268        };
9269        let _ = ask::Questions::open().update(&id, |q| {
9270            q.abandon(why);
9271            Ok(())
9272        });
9273    }
9274
9275    /// Carry a withheld text to its end: file the question, park while it is
9276    /// open, and on an answer, silence or timeout pick the text to post. The
9277    /// decision is saved before it is used, so it applies once.
9278    async fn settle_withheld(
9279        &mut self,
9280        winner: &Candidate,
9281        pr: &PrMessage,
9282        mut g: crate::run::GithubTextAsk,
9283    ) -> Result<Option<PrMessage>> {
9284        use crate::github_text as gt;
9285        let store = ask::Questions::open();
9286        let timeout = self.state.config.graph.answer_timeout;
9287        if !g.resolved {
9288            let mut standing = g
9289                .question
9290                .as_ref()
9291                .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9292            if let Some(q) = standing.as_ref()
9293                && q.status.open()
9294                && !store.held_reply(&q.id)
9295                && gt::expired(q, timeout)
9296            {
9297                let why = format!("no answer within {}s of asking", timeout.max(1));
9298                standing = store
9299                    .update(&q.id, |q| {
9300                        q.abandon(&why);
9301                        Ok(())
9302                    })
9303                    .ok()
9304                    .map(|(q, ())| q);
9305            }
9306            let mut ask_again: Option<Option<Vec<String>>> = None;
9307            match standing {
9308                // Recorded but never written (a stop between the two saves),
9309                // or removed: ask again without spending the retry.
9310                None if g.asks <= 1 => {
9311                    g.asks = 0;
9312                    ask_again = Some(None);
9313                }
9314                None => {
9315                    g.resolved = true;
9316                    g.chosen_title = None;
9317                    g.choice = Some(GithubTextChoice::Fallback);
9318                }
9319                Some(q) if q.status.open() => {
9320                    self.park_on_github_text(&g)?;
9321                    return Ok(None);
9322                }
9323                Some(q) => match gt::read_reply(&q) {
9324                    gt::Reply::Fallback => {
9325                        g.resolved = true;
9326                        g.chosen_title = None;
9327                        g.choice = Some(GithubTextChoice::Fallback);
9328                    }
9329                    gt::Reply::Original => {
9330                        g.resolved = true;
9331                        g.chosen_title = None;
9332                        g.choice = Some(GithubTextChoice::Original);
9333                    }
9334                    gt::Reply::Rewrite => {
9335                        g.resolved = true;
9336                        g.chosen_title = None;
9337                        g.choice = Some(if g.rewrite.is_some() {
9338                            GithubTextChoice::Rewrite
9339                        } else {
9340                            GithubTextChoice::Fallback
9341                        });
9342                    }
9343                    gt::Reply::Title(text) => {
9344                        let first = text
9345                            .lines()
9346                            .map(str::trim)
9347                            .find(|l| !l.is_empty())
9348                            .unwrap_or("");
9349                        let decision = if first.is_empty() {
9350                            None
9351                        } else {
9352                            let candidate = PrMessage {
9353                                title: first.to_owned(),
9354                                body: String::new(),
9355                            };
9356                            self.judge_pr_language(&winner.worktree, &candidate).await
9357                        };
9358                        match gt::vet_title(
9359                            &text,
9360                            decision,
9361                            self.state.config.graph.github_english(),
9362                        ) {
9363                            Ok(title) => {
9364                                g.resolved = true;
9365                                g.chosen_title = Some(title);
9366                                g.choice = Some(GithubTextChoice::MyTitle);
9367                            }
9368                            Err(cats) if g.asks < 2 => {
9369                                ask_again = Some(Some(
9370                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9371                                ))
9372                            }
9373                            Err(_) => {
9374                                g.resolved = true;
9375                                g.chosen_title = None;
9376                                g.choice = Some(GithubTextChoice::Fallback);
9377                            }
9378                        }
9379                    }
9380                },
9381            }
9382            if let Some(rejected) = ask_again {
9383                // A retry (after a rejected replacement title) is still about
9384                // the original text: the same ORIGINAL / FALLBACK comparison.
9385                let w = gt::Withheld {
9386                    title: g.title,
9387                    body: g.body,
9388                    categories: g.categories.clone(),
9389                };
9390                // Named by the original text's fingerprint alone, so a resume
9391                // rewrites the same file and the detail stays identical.
9392                let artifact = match crate::run::write_artifact(
9393                    &self.state,
9394                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9395                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9396                ) {
9397                    Ok(p) => Some(p),
9398                    Err(e) => {
9399                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9400                        None
9401                    }
9402                };
9403                let (detail, comparison) = gt::question_content(
9404                    &w,
9405                    &pr.title,
9406                    &pr.body,
9407                    &self.state.id,
9408                    rejected.as_deref(),
9409                    artifact.as_deref(),
9410                    g.rewrite.as_ref(),
9411                );
9412                let mut q = ask::Question::new(
9413                    self.state.id.clone(),
9414                    gt::ASK_NODE.to_owned(),
9415                    gt::ASK_SEAT.to_owned(),
9416                    gt::question_summary(&self.state.config.graph.language, &w),
9417                    detail,
9418                    gt::question_choices(&w, g.rewrite.as_ref()),
9419                );
9420                q.posting_comparison = Some(comparison);
9421                q.answer_timeout = timeout;
9422                // Recorded before it is written: a stop in between leaves a
9423                // record naming a question the resume then files again.
9424                g.question = Some(q.id.clone());
9425                g.asks += 1;
9426                self.state.github_text = Some(g.clone());
9427                self.state.event(
9428                    "github-text",
9429                    format!("asking the owner about the withheld text ({})", q.short()),
9430                );
9431                self.state.save()?;
9432                store
9433                    .put_executable(&mut q)
9434                    .context("file the posting-gate question")?;
9435                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9436                    tracing::warn!(
9437                        "could not notify about posting-gate question {}: {e:#}",
9438                        q.short()
9439                    );
9440                }
9441                self.park_on_github_text(&g)?;
9442                return Ok(None);
9443            }
9444        }
9445        let resolved = g.chosen_title.clone();
9446        let original = g.choice == Some(GithubTextChoice::Original);
9447        // Field names only: the event log never records the text.
9448        self.state.event(
9449            "github-text",
9450            if original {
9451                format!(
9452                    "posting the original text for: {}",
9453                    match (g.title, g.body) {
9454                        (true, true) => "title, description",
9455                        (true, false) => "title",
9456                        _ => "description",
9457                    }
9458                )
9459            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9460                "posting the saved rewrite title and description".to_owned()
9461            } else if resolved.is_some() {
9462                "posting the owner's replacement title".to_owned()
9463            } else {
9464                "posting the neutral text for the withheld fields".to_owned()
9465            },
9466        );
9467        self.state.github_text = Some(g.clone());
9468        self.state.save()?;
9469        if g.choice == Some(GithubTextChoice::Rewrite)
9470            && let Some(r) = g.rewrite.as_ref()
9471        {
9472            return Ok(Some(PrMessage {
9473                title: r.title.clone(),
9474                body: r.body.clone(),
9475            }));
9476        }
9477        let w = gt::Withheld {
9478            title: g.title,
9479            body: g.body,
9480            categories: g.categories.clone(),
9481        };
9482        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9483        if let Some(t) = resolved {
9484            title = t;
9485        }
9486        // Every withheld field is now neutral or vetted English.
9487        let english = gt::LanguageDecision {
9488            title_english: true,
9489            body_english: true,
9490            body_complete: true,
9491        };
9492        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9493        if original {
9494            // The owner's explicit choice overrides the gate for the fields it
9495            // was made for; the others keep what the gate produced.
9496            if g.title {
9497                title = pr.title.clone();
9498            }
9499            if g.body {
9500                body = pr.body.clone();
9501            }
9502        }
9503        Ok(Some(PrMessage { title, body }))
9504    }
9505
9506    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9507        self.state.github_text = Some(g.clone());
9508        self.state.end_pass_parked(ParkCause::Question);
9509        self.state.parked = true;
9510        self.state.event(
9511            "github-text",
9512            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9513        );
9514        self.state.save()
9515    }
9516
9517    /// Ask `[roles] language_judge` about this text and record which source
9518    /// decided; the text itself is never recorded.
9519    async fn judge_pr_language(
9520        &mut self,
9521        cwd: &Path,
9522        pr: &PrMessage,
9523    ) -> Option<crate::github_text::LanguageDecision> {
9524        self.state.config.roles.language_judge.as_ref()?;
9525        if !self.state.config.graph.language_gate() {
9526            return None;
9527        }
9528        let decision =
9529            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9530        self.state.event(
9531            "github-text",
9532            if decision.is_some() {
9533                "language decided by the language judge"
9534            } else {
9535                "language judge unavailable; using built-in heuristics"
9536            },
9537        );
9538        decision
9539    }
9540
9541    fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9542        Box::pin(self.merge_inner())
9543    }
9544
9545    async fn merge_inner(&mut self) -> Result<()> {
9546        if self.resource_halt()? || self.hold_base_sync().await? {
9547            return Ok(());
9548        }
9549        // Same reasoning as `gate`: ask the review and gate records directly
9550        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9551        // skip can rewrite on reentry to something that no longer says
9552        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9553        // so a hand-off (open findings, green verification) reaches merge
9554        // exactly like a genuinely clean round does.
9555        //
9556        // A run resumed mid-`land` never reaches here at all: `execute`
9557        // recognises `RunStatus::Landing` before it even calls `prep`, and
9558        // routes straight to `run_land` instead. That has to happen a level
9559        // up from this function, not with a check in here, because
9560        // `review_loop`'s own status recomputation (see its doc) runs
9561        // *before* `merge` on every reentry and would otherwise overwrite
9562        // the `Landing` marker with `Gating` before this node ever saw it.
9563        if self
9564            .state
9565            .base_sync
9566            .as_ref()
9567            .is_some_and(|s| s.conflict.is_some())
9568            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9569                != Some(RunStatus::Gating)
9570            // `gate_ran == false` is not "passed" - `gate` leaves it false
9571            // both before it has ever run and when its last attempt was
9572            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9573            // permission to merge on nothing but the review record. Only a
9574            // gate that actually ran may proceed; an intentional local
9575            // review can also finish with verification explicitly not configured.
9576            // `RunState::gate_status` computes that distinction.
9577            || !self.state.gate_allows_progress()
9578        {
9579            return Ok(());
9580        }
9581        // This node's own record, not `status`: `status == Ready` is not
9582        // unique to the harmless `MergeMode::None` path this line was
9583        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9584        // run's PR was closed without merging — and on that run `mode` is
9585        // still `Pr`, so a reentry that fell through here would push and
9586        // open a second pull request. `self.state.merge` is set exactly once
9587        // this node (or `land`) has already produced a verdict, under every
9588        // mode, which is what "already done" actually means here.
9589        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9590            return Ok(());
9591        }
9592        let Some(winner) = self.state.winner().cloned() else {
9593            return Ok(());
9594        };
9595        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9596        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9597            self.state.status = RunStatus::Blocked;
9598            self.state.event(
9599                "gate",
9600                "winner changed after gate; fresh verification required",
9601            );
9602            self.state.save()?;
9603            return Ok(());
9604        }
9605        let wait = self.repository_wait();
9606        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9607            return Ok(());
9608        }
9609        let repo = self.state.repo.clone();
9610        let base = self.state.base_branch.clone();
9611        let mode = self.state.config.merge.mode;
9612        let style = self.state.config.merge.style;
9613        let facts = if is_review_run(&self.state) {
9614            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9615            let start = review_base(
9616                &repo,
9617                &self.state.config.merge.remote,
9618                &base,
9619                &self.state.base_commit,
9620                &winner.branch,
9621            )
9622            .await;
9623            branch_facts(&repo, &start, &winner.branch).await
9624        } else {
9625            None
9626        };
9627        // `None` when the base could not be freshly read: then an adopted
9628        // pull request's title is left alone.
9629        let leaked = if is_review_run(&self.state) {
9630            leaked_subjects(&self.state, &winner.branch).await
9631        } else {
9632            Some(Vec::new())
9633        };
9634        // `None`: parked on the owner's word about a withheld title or body.
9635        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9636        // the resume arrives here again.
9637        let Some(pr) = self
9638            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9639            .await?
9640        else {
9641            return Ok(());
9642        };
9643        let message = pr.commit_message();
9644
9645        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9646        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9647            return Ok(());
9648        }
9649        let outcome = match mode {
9650            MergeMode::None => MergeOutcome {
9651                mode,
9652                ok: true,
9653                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9654                empty: false,
9655            },
9656            MergeMode::Pr | MergeMode::Local
9657                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9658            {
9659                MergeOutcome {
9660                    mode,
9661                    ok: false,
9662                    detail: empty_candidate_detail(&self.state, &base),
9663                    empty: true,
9664                }
9665            }
9666            MergeMode::Local => {
9667                let on = git::current_branch(&repo).await?;
9668                if on.as_deref() != Some(base.as_str()) {
9669                    MergeOutcome {
9670                        mode,
9671                        ok: false,
9672                        detail: format!(
9673                            "{} has {} checked out, not the base branch {base}",
9674                            repo.display(),
9675                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9676                        ),
9677                        empty: false,
9678                    }
9679                } else if !git::is_clean(&repo).await? {
9680                    MergeOutcome {
9681                        mode,
9682                        ok: false,
9683                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9684                        empty: false,
9685                    }
9686                } else {
9687                    let out = match style {
9688                        MergeStyle::Merge => {
9689                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9690                        }
9691                        MergeStyle::Squash => {
9692                            git::merge_squash(&repo, &winner.branch, &message).await?
9693                        }
9694                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9695                    };
9696                    MergeOutcome {
9697                        mode,
9698                        ok: out.ok(),
9699                        detail: if out.ok() { out.stdout } else { out.stderr },
9700                        empty: false,
9701                    }
9702                }
9703            }
9704            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9705                self.state.pr_publication_pending = true;
9706                self.state.save()?;
9707                let out =
9708                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9709                self.state.pr_publication_pending = out.is_err();
9710                MergeOutcome {
9711                    mode,
9712                    ok: out.is_ok(),
9713                    detail: match out {
9714                        Ok(url) => url,
9715                        Err(e) => format!("{e:#}"),
9716                    },
9717                    empty: false,
9718                }
9719            }
9720            MergeMode::Pr => {
9721                let remote = self.state.config.merge.remote.clone();
9722                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9723                let pushed = if self.state.foreign_checkout.is_some() {
9724                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9725                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9726                } else {
9727                    git::push(&winner.worktree, &remote, &winner.branch).await?
9728                };
9729                if !pushed.ok() {
9730                    MergeOutcome {
9731                        mode,
9732                        ok: false,
9733                        detail: pushed.stderr,
9734                        empty: false,
9735                    }
9736                } else {
9737                    if self.state.foreign_checkout.is_some() {
9738                        self.state.pr_pushed_head = Some(delivery_head.clone());
9739                    }
9740                    // A retry or resume of a run whose branch already has an
9741                    // open pull request adopts it rather than failing on a
9742                    // duplicate. Only this winner branch into this base:
9743                    // `branch_for` derives the name from the run id, so a
9744                    // different run's pull request never matches.
9745                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9746                    let out = match pr_merge_plan(found) {
9747                        PrPlan::Create => {
9748                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9749                                .await
9750                        }
9751                        PrPlan::Adopt { url, title } => {
9752                            self.state
9753                                .event("merge", format!("Pr: adopted open pull request {url}"));
9754                            if self.state.explicit_github_text_selection() {
9755                                // An explicit publication selection applies the prepared
9756                                // payload; acceptance alone is not publication proof.
9757                                land::set_pr_text_verbatim(
9758                                    &winner.worktree,
9759                                    &url,
9760                                    &pr.title,
9761                                    &pr.body,
9762                                )
9763                                .await
9764                                .map(|()| url)
9765                            } else {
9766                                if title != pr.title
9767                                    && (!is_review_run(&self.state)
9768                                        || leaked
9769                                            .as_deref()
9770                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9771                                    && let Err(e) =
9772                                        if self.state.github_text.as_ref().is_some_and(|g| {
9773                                            g.resolved
9774                                                && g.title
9775                                                && g.choice == Some(GithubTextChoice::Original)
9776                                        }) {
9777                                            // The owner chose to post this title unmodified.
9778                                            land::set_pr_title_verbatim(
9779                                                &winner.worktree,
9780                                                &url,
9781                                                &pr.title,
9782                                            )
9783                                            .await
9784                                        } else {
9785                                            land::set_pr_title(
9786                                                &mut self.state,
9787                                                &winner.worktree,
9788                                                &url,
9789                                                &pr.title,
9790                                            )
9791                                            .await
9792                                        }
9793                                {
9794                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9795                                    self.state
9796                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9797                                }
9798                                Ok(url)
9799                            }
9800                        }
9801                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9802                    };
9803                    match out {
9804                        Ok(url) => MergeOutcome {
9805                            mode,
9806                            ok: true,
9807                            detail: url,
9808                            empty: false,
9809                        },
9810                        Err(e) => MergeOutcome {
9811                            mode,
9812                            ok: false,
9813                            detail: e.to_string(),
9814                            empty: false,
9815                        },
9816                    }
9817                }
9818            }
9819        };
9820
9821        self.state.status = match (mode, outcome.ok) {
9822            (MergeMode::None, _) => RunStatus::Ready,
9823            (_, true) if outcome.empty => RunStatus::Ready,
9824            (MergeMode::Pr, true) => RunStatus::Ready,
9825            (_, true) => RunStatus::Merged,
9826            (_, false) => RunStatus::Blocked,
9827        };
9828        self.state.event(
9829            "merge",
9830            format!(
9831                "{:?}: {}",
9832                mode,
9833                outcome.detail.lines().next().unwrap_or("")
9834            ),
9835        );
9836        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9837            self.state.record_github_text_publication(&outcome.detail);
9838        }
9839        self.state.merge = Some(outcome);
9840        self.state.save()?;
9841
9842        // The PR is open and the run would historically stop here, leaving the
9843        // operator to watch checks, feed review comments back to a fixer, and
9844        // merge. That was done by hand six times in one session before this
9845        // existed. Opt-in, because merging is the one irreversible thing magi
9846        // can do to a repository.
9847        if self.state.config.graph.land
9848            && mode == MergeMode::Pr
9849            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9850        {
9851            self.run_land().await?;
9852        }
9853        // `run_land` may have left `status` at `Landing` - still waiting on
9854        // CI or the owner's approval, not actually settled - so this has to
9855        // read whatever `status` ended up as here, not the `Merged` this
9856        // function set a few lines up.
9857        self.settle_questions();
9858        Ok(())
9859    }
9860
9861    /// Enter `land`.
9862    ///
9863    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9864    /// a resumed run's re-entry. `land::land` itself is what serialises the
9865    /// two git-mutating moments inside the loop — the rebase push and
9866    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9867    /// to hold a lock across the whole call, and doing so would serialise
9868    /// this run's CI wait against a *different* run's land-approval resume
9869    /// in the same repository, which is exactly the "must not wait on
9870    /// another task" property the daemon's slot-freeing exists to give.
9871    fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9872        Box::pin(self.run_land_inner())
9873    }
9874
9875    async fn run_land_inner(&mut self) -> Result<()> {
9876        crate::pr_head::ensure_owner(&self.state)?;
9877        let url = self
9878            .state
9879            .pr
9880            .as_ref()
9881            .map(|pr| pr.url.clone())
9882            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9883            .unwrap_or_default();
9884        let url = url.lines().next().unwrap_or("").trim().to_owned();
9885        if !url.starts_with("http") {
9886            return Ok(());
9887        }
9888        // A land failure is not a lost run: the work is on a branch and the
9889        // pull request is open, which is exactly where a human takes over.
9890        loop {
9891            match land::land(&mut self.state, &url).await {
9892                Ok(_)
9893                    if self
9894                        .state
9895                        .merge_repairs
9896                        .last()
9897                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9898                {
9899                    if self.repair_merge_review().await? {
9900                        continue;
9901                    }
9902                }
9903                Ok(_) if self.state.resource_block.is_some() => {
9904                    return self.stop_for_resource();
9905                }
9906                Ok(pr) if self.state.parked => {
9907                    // `land` already saved the parked marker; nothing here
9908                    // overrides `status` back to a terminal value while an
9909                    // approval is still outstanding.
9910                    let _ = pr;
9911                }
9912                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9913                    self.state.status = RunStatus::Superseded;
9914                    self.state.save()?;
9915                }
9916                Ok(pr) => {
9917                    self.state.status = match pr.state {
9918                        land::PrLifecycle::Merged => RunStatus::Merged,
9919                        _ => RunStatus::Blocked,
9920                    };
9921                    // Downstream of a confirmed merge only - see
9922                    // `bump::should_release_bump`'s own doc for why this one
9923                    // check covers all three of `land`'s success paths.
9924                    // Best-effort: the run already landed, so a failure here
9925                    // (the decision call, `gh`, `cargo`) is recorded and never
9926                    // turns a landed run into a failed one.
9927                    if bump::should_release_bump(self.state.status)
9928                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9929                    {
9930                        // The event is the run's own record. Not-eligible cases
9931                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9932                        // `Err` is a bump that was tried and failed:
9933                        // `after_merge` itself raises the operator notice for
9934                        // that, whether or not a release PR exists yet.
9935                        self.state
9936                            .event("bump", format!("release bump skipped: {e:#}"));
9937                    }
9938                    // Independent of the bump, and best-effort in the same way:
9939                    // findings the merge left open become follow-up tasks.
9940                    if self.state.status == RunStatus::Merged {
9941                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9942                    }
9943                    self.state.save()?;
9944                }
9945                Err(e) => {
9946                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9947                        return Err(e);
9948                    }
9949                    self.state.status = RunStatus::Blocked;
9950                    self.state.event("land", format!("gave up: {e}"));
9951                    self.state.save()?;
9952                }
9953            }
9954            break;
9955        }
9956        Ok(())
9957    }
9958
9959    /// One repair cycle stays under the existing driver/task claim. An
9960    /// interrupted fixer is rolled back and held rather than run twice.
9961    async fn repair_merge_review(&mut self) -> Result<bool> {
9962        use crate::run::MergeRepairPhase as Phase;
9963        crate::pr_head::ensure_owner(&self.state)?;
9964        let i = self.state.merge_repairs.len() - 1;
9965        let request = self.state.merge_repairs[i].clone();
9966        if request.phase == Phase::Fixing {
9967            let restored = match self.state.winner().cloned() {
9968                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9969                None => Err(anyhow::anyhow!("repair has no winning branch")),
9970            };
9971            let why = match restored {
9972                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9973                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9974                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9975            };
9976            self.hold_merge_repair(i, why)?;
9977            return Ok(false);
9978        }
9979        // A completed repair gate is a publication checkpoint. On reentry,
9980        // reject external changes before the ordinary gate path can replace
9981        // that evidence by verifying a new head. Requested repairs still
9982        // start their own review generation below.
9983        if request.phase == Phase::Reviewing && self.state.gate_ran {
9984            let winner = self
9985                .state
9986                .winner()
9987                .context("repair has no winning branch")?;
9988            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9989            if self.state.gate_head.as_deref() != Some(head.as_str())
9990                || !git::is_clean(&winner.worktree).await?
9991            {
9992                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9993                return Ok(false);
9994            }
9995        }
9996        let failure = if i >= self.state.config.graph.land_rounds
9997            || self.state.config.graph.review_rounds == 0
9998        {
9999            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
10000        } else {
10001            None
10002        };
10003        if let Some(why) = failure {
10004            self.hold_merge_repair(i, why)?;
10005            return Ok(false);
10006        }
10007        if request.phase == Phase::Requested {
10008            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
10009                Ok(pr) => pr,
10010                Err(e) => {
10011                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
10012                    return Ok(false);
10013                }
10014            };
10015            self.state.merge_repairs[i].phase = Phase::Fixing;
10016            self.state.land_approval = None;
10017            self.state.land_armed_head = None;
10018            self.state.save()?;
10019            // Serialize the exact records, including finding ids/details,
10020            // reconsideration reasons, final votes and verification output.
10021            let evidence = serde_json::to_string_pretty(&request)?;
10022            let reason = format!(
10023                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
10024            );
10025            let budget = self.state.config.graph.land_rounds;
10026            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10027            match result {
10028                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10029                Ok(land::Fixed::Deferred) => {
10030                    // An unavailable fixer did not spend this repair request.
10031                    // Restore partial fallback work before retrying it, as for
10032                    // an interrupted repair, without closing the request.
10033                    let worktree = self
10034                        .state
10035                        .winner()
10036                        .context("repair has no winning branch")?
10037                        .worktree
10038                        .clone();
10039                    self.graft_restore(&worktree, &request.head).await?;
10040                    self.state.merge_repairs[i].phase = Phase::Requested;
10041                    self.stop_for_resource()?;
10042                    return Ok(false);
10043                }
10044                Ok(land::Fixed::Failed(why)) => {
10045                    self.hold_merge_repair(
10046                        i,
10047                        format!(
10048                            "repair failed: {why}; inspect the branch and run an explicit review"
10049                        ),
10050                    )?;
10051                    return Ok(false);
10052                }
10053                Err(e) => {
10054                    self.hold_merge_repair(
10055                        i,
10056                        format!(
10057                            "repair failed: {e:#}; inspect the branch and run an explicit review"
10058                        ),
10059                    )?;
10060                    return Ok(false);
10061                }
10062            }
10063            // Even no change gets a real new reviewer wave. The previous
10064            // evidence remains in the request; a former clean/concluded round
10065            // cannot short-circuit the new review or gate.
10066            self.begin_merge_repair_review(i)?;
10067        }
10068        self.review_loop().await?;
10069        self.gate().await?;
10070        if self.resource_halt()? || self.state.parked {
10071            return Ok(false);
10072        }
10073        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10074            != Some(RunStatus::Gating)
10075            || !self.state.gate_allows_progress()
10076        {
10077            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10078            return Ok(false);
10079        }
10080        // Review and gate may have added commits after the initial repair.
10081        // Publish exactly their final head to the existing PR, with a lease.
10082        let winner = self
10083            .state
10084            .winner()
10085            .cloned()
10086            .context("repair has no winning branch")?;
10087        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10088        if self.state.gate_head.as_deref() != Some(after.as_str())
10089            || !git::is_clean(&winner.worktree).await?
10090        {
10091            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10092            return Ok(false);
10093        }
10094        let expected = self
10095            .state
10096            .pr_pushed_head
10097            .as_deref()
10098            .unwrap_or(&request.head)
10099            .to_owned();
10100        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10101        // Push the checked commit itself: a local ref moving after the check
10102        // must not change which commit the forge receives.
10103        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10104            return Ok(false);
10105        }
10106        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10107        let refspec = format!("{after}:refs/heads/{public}");
10108        let push = git::git_raw(
10109            &winner.worktree,
10110            &["push", &lease, &self.state.config.merge.remote, &refspec],
10111        )
10112        .await;
10113        match push {
10114            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10115            Ok(out) => {
10116                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10117                return Ok(false);
10118            }
10119            Err(e) => {
10120                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10121                return Ok(false);
10122            }
10123        }
10124        self.state.merge_repairs[i].phase = Phase::Complete;
10125        self.state.status = RunStatus::Landing;
10126        self.state.event(
10127            "repair",
10128            "new review and gate completed; fresh owner merge decision required",
10129        );
10130        self.state.save()?;
10131        Ok(true)
10132    }
10133
10134    fn merge_repair_artifacts(&self) -> PathBuf {
10135        let artifacts = agent::artifacts_dir(&self.state.dir());
10136        match self
10137            .state
10138            .merge_repairs
10139            .last()
10140            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10141        {
10142            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10143            None => artifacts,
10144        }
10145    }
10146
10147    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10148        if let Some(e2e) = self.state.head_e2e.take() {
10149            self.state
10150                .verification_history
10151                .push(crate::run::VerificationEvidence {
10152                    superseded_by: None,
10153                    archived_at: Some(Timestamp::now()),
10154                    fixes_spent: Some(self.state.gate_fixes.len()),
10155                    gate_head: None,
10156                    gate_ran: false,
10157                    gate: Vec::new(),
10158                    e2e: Some(e2e),
10159                });
10160        }
10161        self.state.reviews.clear();
10162        self.state.gate.clear();
10163        self.state.gate_ran = false;
10164        self.state.gate_head = None;
10165        self.state.gate_fixes.clear();
10166        self.state.gate_repair_e2e = None;
10167        self.state.pre_gate.clear();
10168        self.state.pre_gate_commit = None;
10169        self.state.contested_handoff = None;
10170        self.state.land_reruns = None;
10171        self.state.land_approval = None;
10172        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10173        self.state.status = RunStatus::Reviewing;
10174        self.state.save()
10175    }
10176
10177    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10178        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10179        self.state.merge_repairs[index].failure = Some(why.clone());
10180        self.state.status = RunStatus::Blocked;
10181        self.state.event("repair", why);
10182        self.state.save()
10183    }
10184
10185    // -------------------------------------------------------------- helpers
10186
10187    /// Fetch or create a seat, keeping its conversation across nodes.
10188    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10189        if let Some(existing) = self.state.seats.get(key)
10190            && existing.agent == agent
10191        {
10192            return existing.clone();
10193        }
10194        // A seat that changes agent mints its session id from the agent too,
10195        // like a handover: the old agent's uuid is already taken by the CLI.
10196        let fresh = if self.state.seats.contains_key(key) {
10197            handover_seat(key, agent, self.state.next_seat_seed())
10198        } else {
10199            SeatState::new(key, agent, self.state.seed)
10200        };
10201        self.state.seats.insert(key.to_owned(), fresh.clone());
10202        fresh
10203    }
10204
10205    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10206    /// seat to another roster agent, in which case that agent. Nodes that
10207    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10208    /// reconsideration) must keep talking to whoever answered it, not slip
10209    /// back to the agent that failed it.
10210    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10211        match self.state.seats.get(key) {
10212            Some(s) if s.agent != spec.id => {
10213                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10214            }
10215            _ => spec,
10216        }
10217    }
10218
10219    /// A candidate rendered for judging, with the leak policy applied.
10220    fn view(&self, c: &Candidate) -> CandidateView {
10221        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10222            .unwrap_or_default();
10223        let (patch, _) = blind::sanitize_patch(
10224            &format!("candidate {} patch", c.label),
10225            &raw,
10226            &self.state.config.blind,
10227        );
10228        CandidateView {
10229            label: c.label,
10230            branch: c.branch.clone(),
10231            summary: c.summary.clone(),
10232            stat: c.stat.clone(),
10233            patch,
10234        }
10235    }
10236
10237    /// The full candidate set as prompt text, for seats with no live session.
10238    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10239        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10240        prompt::judge(
10241            "(see above)",
10242            &views,
10243            self.roles.judges.len(),
10244            base_short,
10245            "en",
10246        )
10247    }
10248
10249    /// The final-vote prompt with everything a seat that has no session of its
10250    /// own needs: the candidates, the seat's own ranking and reasons, and the
10251    /// anonymised deliberation it took part in (only when there was one, so a
10252    /// handed-over seat never sees more than the seat it replaces did). The
10253    /// `Final vote` heading stays first.
10254    fn vote_prompt_full(
10255        &self,
10256        j: usize,
10257        labels: &[char],
10258        language: &str,
10259        candidates: &[Candidate],
10260        base_short: &str,
10261    ) -> String {
10262        let mut text = format!(
10263            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10264            prompt::final_vote(labels, language),
10265            self.state.instruction,
10266            self.candidate_block(candidates, base_short)
10267        );
10268        if let Some(own) = self
10269            .state
10270            .judgements
10271            .get(j)
10272            .filter(|r| !r.ranking.is_empty())
10273        {
10274            let reasons = own
10275                .reasons
10276                .iter()
10277                .map(|(k, v)| format!("- {k}: {v}"))
10278                .collect::<Vec<_>>()
10279                .join("\n");
10280            text.push_str(&format!(
10281                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10282                own.ranking.iter().collect::<String>(),
10283                if reasons.is_empty() {
10284                    ""
10285                } else {
10286                    ", because:\n"
10287                }
10288            ));
10289        }
10290        if !self.state.deliberation.is_empty() {
10291            text.push_str("\n# What was argued before this vote\n");
10292            for t in self.transcript(&[], j) {
10293                text.push_str(&format!(
10294                    "\n## {}{}\n\n{}\n",
10295                    t.who,
10296                    if t.is_self { " (you)" } else { "" },
10297                    t.body.trim()
10298                ));
10299            }
10300        }
10301        text
10302    }
10303
10304    /// Anonymised transcript for judge `self_idx`.
10305    ///
10306    /// The initial rankings are always the opening statements. Seeding them
10307    /// only when no turn had been taken yet meant every judge after the first
10308    /// argued against a single voice instead of against the actual split — the
10309    /// disagreement is the information, so it is always on the table.
10310    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10311        let mut turns = Vec::new();
10312        for j in &self.state.judgements {
10313            if j.ranking.is_empty() {
10314                continue;
10315            }
10316            let reasons = j
10317                .reasons
10318                .iter()
10319                .map(|(k, v)| format!("- {k}: {v}"))
10320                .collect::<Vec<_>>()
10321                .join("\n");
10322            turns.push(Turn {
10323                who: format!("Judge {} (opening ranking)", j.judge),
10324                is_self: j.judge == self_idx + 1,
10325                body: format!(
10326                    "Ranked {}{}{reasons}",
10327                    j.ranking.iter().collect::<String>(),
10328                    if reasons.is_empty() {
10329                        ""
10330                    } else {
10331                        ", because:\n"
10332                    }
10333                ),
10334            });
10335        }
10336        for t in self
10337            .state
10338            .deliberation
10339            .iter()
10340            .flat_map(|r| r.turns.iter())
10341            .chain(current)
10342        {
10343            turns.push(Turn {
10344                who: format!("Judge {}", t.judge),
10345                is_self: t.judge == self_idx + 1,
10346                body: t.body.clone(),
10347            });
10348        }
10349        turns
10350    }
10351}
10352
10353/// Does this seat still hold the context a follow-up prompt would rely on?
10354fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10355    agent::has_session(spec.kind, seat, sessions)
10356}
10357
10358/// The next entry in `roster` after `start`, never wrapping back to the
10359/// front, whose id is not in `tried` yet.
10360///
10361/// Starts one past `start` rather than at the front of `roster`: `start` is
10362/// the seat's own original position, and a seat whose candidate slot already
10363/// sits on the roster's second entry must fall through to the third next, not
10364/// restart at the first — which is very likely a different candidate's own
10365/// agent already. Never wraps back past `start`, for the same reason: an
10366/// entry earlier in the roster than the seat's own position is almost
10367/// certainly some *other* candidate slot's own agent, and once the tail of
10368/// the roster is exhausted there are no more untried agents for *this* seat
10369/// to fall through to — the caller's fallback chain ends there, exactly as
10370/// "no further untried agents remain in the list for that seat" asks for.
10371///
10372/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10373/// the same id twice (an operator's `roles.implementers` typo, or a
10374/// `[[agents]]` list reused across roles) must not let
10375/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10376/// over `roster` either finds an untried id or runs out, so this always
10377/// terminates regardless of duplicates.
10378fn next_untried_in_roster<'a>(
10379    roster: &'a [AgentSpec],
10380    start: usize,
10381    tried: &BTreeSet<String>,
10382) -> Option<&'a AgentSpec> {
10383    roster
10384        .get(start + 1..)?
10385        .iter()
10386        .find(|s| !tried.contains(&s.id))
10387}
10388
10389/// The successor for a seat, shared by all four seat kinds (implement, judge,
10390/// review, advise). `others` holds the ids that *currently* occupy the other
10391/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10392/// sees them), so roster entries beyond the seat count act as spares: a failed
10393/// seat goes to an agent no other seat holds whenever the roster permits.
10394///
10395/// `carried` is `Some` only for the review loop's carried failure history.
10396/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10397/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10398/// whole roster: untried and not another seat's occupant; (3) the plain walk
10399/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10400/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10401/// a better outcome than an empty seat, and it keeps the answer to "is there
10402/// a successor at all" exactly what it was before occupants were considered,
10403/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10404/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10405/// carried failure is a cheaper bet than doubling an agent on the panel, and
10406/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10407/// the same round are handled one at a time, so a seat later in the batch
10408/// sees an earlier one's new occupant but not yet its own freed agent.
10409fn pick_successor<'a>(
10410    roster: &'a [AgentSpec],
10411    start: usize,
10412    tried: &BTreeSet<String>,
10413    carried: Option<&BTreeSet<String>>,
10414    others: &BTreeSet<String>,
10415) -> Option<&'a AgentSpec> {
10416    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10417    roster
10418        .get(start + 1..)
10419        .and_then(|tail| {
10420            tail.iter()
10421                .filter(free)
10422                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10423        })
10424        .or_else(|| {
10425            carried
10426                .is_some()
10427                .then(|| roster.iter().find(free))
10428                .flatten()
10429        })
10430        .or_else(|| match carried {
10431            Some(c) => next_for_seat(roster, start, tried, c),
10432            None => next_untried_in_roster(roster, start, tried),
10433        })
10434}
10435
10436/// The next agent for a seat that carries its failure history across rounds
10437/// (the review loop). `round_tried` is this round's own bound and starts
10438/// empty every round; `carried_failed` only decides priority.
10439///
10440/// First: an id walking forward from `start`, never wrapping, that is neither
10441/// tried this round nor failed in an earlier one. Only when that is exhausted
10442/// does it rescue: the first roster id (in roster order, so this one *does*
10443/// look before `start`) not yet tried this round, which is by then a carried
10444/// failure. Each id is rescued at most once per round, so it cannot loop.
10445fn next_for_seat<'a>(
10446    roster: &'a [AgentSpec],
10447    start: usize,
10448    round_tried: &BTreeSet<String>,
10449    carried_failed: &BTreeSet<String>,
10450) -> Option<&'a AgentSpec> {
10451    roster
10452        .get(start + 1..)?
10453        .iter()
10454        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10455        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10456}
10457
10458/// Where a reviewer seat starts a round: the agent that last answered it when
10459/// it is still on the roster and not marked failed, else the spec's own agent
10460/// unless it failed, else the next roster agent that has not failed, else the
10461/// spec's own agent again (the whole roster failed). Ids no longer on the
10462/// roster are ignored. An empty roster has no handover, so the spec stands.
10463fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10464    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10465        return spec;
10466    };
10467    let ok = |id: &str| !h.failed.contains(id);
10468    if let Some(last) = h.last_ok.as_deref()
10469        && ok(last)
10470        && let Some(s) = roster.iter().find(|s| s.id == last)
10471    {
10472        return s.clone();
10473    }
10474    if ok(&spec.id) {
10475        return spec;
10476    }
10477    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10478    roster
10479        .get(from + 1..)
10480        .into_iter()
10481        .flatten()
10482        .chain(roster.iter())
10483        .find(|s| ok(&s.id))
10484        .cloned()
10485        .unwrap_or(spec)
10486}
10487
10488/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10489/// seed so a CLI that mints its session id up front (`--session-id`) never
10490/// reuses the uuid the previous agent already opened under the same seat key.
10491pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10492    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10493}
10494
10495/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10496/// so the classifier below and the code that builds the message cannot drift.
10497const TIMED_OUT: &str = "timed out";
10498
10499impl FailClass {
10500    /// `None` for an answer; otherwise how the turn failed.
10501    fn of(out: &AgentOutcome) -> Option<Self> {
10502        match out {
10503            AgentOutcome::Ok(_) => None,
10504            AgentOutcome::Quota(_) => Some(Self::Quota),
10505            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10506            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10507            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10508            // Not a failure of the agent. Callers check for it first; `None`
10509            // here means "no class to hand over on".
10510            AgentOutcome::ResourceBlocked(_) => None,
10511        }
10512    }
10513
10514    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10515    fn stem_word(&self) -> &'static str {
10516        match self {
10517            Self::Quota => "quota",
10518            _ => "handover",
10519        }
10520    }
10521}
10522
10523/// The message's first line with its variable parts removed — digit runs and
10524/// path-like tokens — so "exit code 2" and "exit code 7" read
10525/// as one kind of failure.
10526fn failure_signature(msg: &str) -> String {
10527    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10528    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10529    // the historical machine key while the process diagnosis uses readable prose.
10530    if let Some(code) = line
10531        .strip_prefix("process exited with no usable output (exit code ")
10532        .and_then(|s| s.strip_suffix(')'))
10533        .and_then(|s| s.parse::<i32>().ok())
10534    {
10535        line = format!("exited with some({code}) and no usable output");
10536    } else if matches!(
10537        line.as_str(),
10538        "process exited with no usable output (no exit code reported)"
10539            | "process exited with no usable output (exit code unavailable)"
10540    ) {
10541        line = "exited with none and no usable output".to_owned();
10542    }
10543    let mut out = Vec::new();
10544    for word in line.split_whitespace() {
10545        if word.contains('/') || word.contains('\\') {
10546            out.push("<path>".to_owned());
10547            continue;
10548        }
10549        let mut w = String::new();
10550        let mut in_digits = false;
10551        for c in word.chars() {
10552            if c.is_ascii_digit() {
10553                if !in_digits {
10554                    w.push('#');
10555                }
10556                in_digits = true;
10557            } else {
10558                in_digits = false;
10559                w.push(c);
10560            }
10561        }
10562        out.push(w);
10563    }
10564    out.join(" ").chars().take(120).collect()
10565}
10566
10567/// Whether a seat that just failed with `cur` may go to the next roster agent.
10568/// A quota or a timeout always may. Any other failure may not when the agent
10569/// before it failed the same way: an error the prompt causes would otherwise
10570/// walk the whole roster. `prev` is the class of the immediately preceding
10571/// agent's failure, so a quota or timeout in between breaks the run of
10572/// identical failures by itself.
10573fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10574    match cur {
10575        FailClass::Quota | FailClass::Timeout => true,
10576        FailClass::Other(_) => prev != Some(cur),
10577    }
10578}
10579
10580/// A short human reason for a failed outcome, for the handover record.
10581fn fail_reason(out: &AgentOutcome) -> String {
10582    match out {
10583        AgentOutcome::Ok(_) => String::new(),
10584        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10585        AgentOutcome::Dropped(o) => format!(
10586            "the CLI dropped the stream ({})",
10587            o.dropped
10588                .as_ref()
10589                .map(|d| d.why.as_str())
10590                .unwrap_or("it ended without delivering its answer")
10591        ),
10592        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10593            e.lines().next().unwrap_or("").chars().take(160).collect()
10594        }
10595    }
10596}
10597
10598/// Note one handover in the run: the structured record and, in the timeline,
10599/// the sentence a person reads. A quota keeps the wording it always had.
10600pub(crate) fn record_handover(
10601    state: &mut RunState,
10602    node: &str,
10603    seat: &str,
10604    from: &str,
10605    to: &str,
10606    class: &FailClass,
10607    reason: &str,
10608) {
10609    let message = if *class == FailClass::Quota {
10610        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10611    } else {
10612        format!("{seat}: handed over {from} -> {to} ({reason})")
10613    };
10614    state.event(node, message);
10615    state.handovers.push(Handover {
10616        at: Timestamp::now(),
10617        node: node.to_owned(),
10618        seat: seat.to_owned(),
10619        from: from.to_owned(),
10620        to: to.to_owned(),
10621        reason: reason.to_owned(),
10622    });
10623}
10624
10625/// Did this reply report running a command whose own CLI never confirmed an
10626/// exit status?
10627///
10628/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10629/// command *finished* (see that type's own doc), so this can only be `true`
10630/// for a command whose completion event carried no readable exit code — not
10631/// for one that simply is not mentioned at all. That is the one signal this
10632/// crate can read, from the same record `state.jobs` renders, about a reply
10633/// standing next to work its own CLI cannot vouch for finishing; it is
10634/// deliberately not a check on the exit code's *value* (a fixer legitimately
10635/// runs a command that fails mid-iteration before it succeeds) and not a
10636/// guess at a command still running in the background (which emits no event
10637/// at all, and so leaves no evidence here to find).
10638fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10639    commands.iter().any(|c| c.exit_code.is_none())
10640}
10641
10642/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10643/// trusted as a verified no-op — the adoption guard's own text-level half.
10644///
10645/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10646/// timed out): a marker only earns the benefit of the doubt from a turn the
10647/// CLI itself vouches for finishing properly, the same house style
10648/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10649/// *fix* report to for `commands`. A candidate that timed out, exited
10650/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10651/// is, whatever prose it wrote — this returns `None` before it ever looks at
10652/// `text`. The remaining guards (the tree really is empty, the evidence is
10653/// non-empty) are the caller's: this only reads what the reply *claimed*.
10654fn verified_noop_claim(
10655    usable: bool,
10656    commands: &[agent::CommandEvidence],
10657    text: &str,
10658) -> Option<String> {
10659    (usable && !has_unconfirmed_command(commands))
10660        .then(|| verdict::verified_noop(text))
10661        .flatten()
10662}
10663
10664fn short(commit: &str) -> String {
10665    commit.chars().take(7).collect()
10666}
10667
10668fn make_executable(path: &Path) -> Result<()> {
10669    #[cfg(unix)]
10670    {
10671        use std::os::unix::fs::PermissionsExt as _;
10672        let mut perms = std::fs::metadata(path)?.permissions();
10673        perms.set_mode(0o755);
10674        std::fs::set_permissions(path, perms)?;
10675    }
10676    #[cfg(not(unix))]
10677    {
10678        let _ = path;
10679    }
10680    Ok(())
10681}
10682
10683/// What every seat in one batch shares: where the answers are attributed, the
10684/// prompt overlay they inherit, and the build cache they are told to use.
10685///
10686/// A struct rather than four more parameters: `wave` also needs the run's
10687/// state (to record who is answering right now) and the attempt number, and
10688/// eight positional arguments is both unreadable and a clippy error.
10689struct WaveCtx<'a> {
10690    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10691    /// paid for it.
10692    run: &'a str,
10693    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10694    node: &'a str,
10695    prompts: &'a Prompts,
10696    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10697    cache: Option<&'a Path>,
10698    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10699    /// `JobRecord::round`. `None` for every other node.
10700    round: Option<usize>,
10701    /// Carry each seat's failed-agent history across waves (the review loop
10702    /// only): start-of-round priority and handover choice read
10703    /// [`RunState::seat_history`], and every answer or failure writes it.
10704    carry_seats: bool,
10705    /// The run's park / interrupt requests, so a wait for a shared resource
10706    /// can be abandoned the moment the operator asks to park.
10707    pauses: Option<(&'a Pause, &'a Pause)>,
10708}
10709
10710/// Shared scheduling policy for operations and spawned seat invocations.
10711fn repository_wait(
10712    state: &RunState,
10713    pauses: Option<(&Pause, &Pause)>,
10714) -> crate::repo_lock::WaitPolicy {
10715    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10716    let inherited = crate::repo_lock::WaitPolicy::current();
10717    crate::repo_lock::WaitPolicy::new(
10718        Duration::from_secs(state.config.graph.timeout_resource_wait),
10719        move || {
10720            inherited.cancelled()
10721                || pauses
10722                    .as_ref()
10723                    .is_some_and(|(a, b)| a.parked() || b.parked())
10724        },
10725    )
10726}
10727
10728fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10729    if error.is::<crate::repo_lock::ResourceBlocked>() {
10730        state.resource_block = Some(crate::run::ResourceBlock {
10731            cause: crate::run::ResourceBlockCause::Other,
10732            node: "repository".into(),
10733            kind: if error.to_string().contains("disk pressure") {
10734                "disk pressure"
10735            } else if error.to_string().contains("disk admission") {
10736                "disk admission"
10737            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10738                "incremental cache admission"
10739            } else {
10740                "repository coordination"
10741            }
10742            .into(),
10743            detail: error.to_string(),
10744            at: Timestamp::now(),
10745        });
10746        state.event("resource", error.to_string());
10747    }
10748}
10749
10750async fn coordinate_operation(
10751    state: &mut RunState,
10752    cwd: &Path,
10753    wait: &crate::repo_lock::WaitPolicy,
10754) -> Result<Option<crate::repo_lock::SeatLease>> {
10755    let result =
10756        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10757            .await;
10758    if let Err(e) = &result {
10759        record_repository_block(state, e);
10760    }
10761    result
10762}
10763
10764/// Run one job, honouring the parallelism budget.
10765async fn run_one(
10766    job: SeatJob,
10767    sem: Arc<Semaphore>,
10768    ctx: &WaveCtx<'_>,
10769    state: &mut RunState,
10770    attempt: usize,
10771) -> (SeatState, AgentOutcome) {
10772    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10773        .await
10774        .pop()
10775        .expect("one job in, one result out");
10776    (seat, out)
10777}
10778
10779/// Run every job concurrently, capped by the semaphore, preserving order.
10780///
10781/// Every seat in the batch is recorded into [`RunState::active`] before the
10782/// wave starts and cleared as each answer lands, so the run's own record says
10783/// who is still being waited on rather than only who finished.
10784async fn wave(
10785    jobs: Vec<SeatJob>,
10786    sem: Arc<Semaphore>,
10787    ctx: &WaveCtx<'_>,
10788    state: &mut RunState,
10789    attempt: usize,
10790) -> Vec<(usize, SeatState, AgentOutcome)> {
10791    let WaveCtx {
10792        run,
10793        node,
10794        prompts,
10795        cache,
10796        round,
10797        carry_seats: _,
10798        pauses,
10799    } = *ctx;
10800    for job in &jobs {
10801        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10802    }
10803    if let Err(e) = state.save() {
10804        // A failed persist of "who is answering right now" must not abort the
10805        // wave: the seats are already being asked, and the alternative is
10806        // losing the answers to save a status line nobody may even be
10807        // watching.
10808        tracing::warn!("could not persist in-progress seats: {e:#}");
10809    }
10810    // Hold the shared build cache's lease for the whole batch, not per job:
10811    // several candidates (an implement wave) or a fixer legitimately share
10812    // one cache concurrently within this run, and that stays untouched — a
10813    // single lease taken once for the whole wave and released once it is
10814    // done is what stops a *different* borrower (another run's own wave, its
10815    // e2e/gate, a human's `magi review`) from interleaving a build into the
10816    // same directory while this one is in flight. Best-effort, not
10817    // all-or-nothing: a wave that cannot get the lease within its own
10818    // longest job's budget still runs — an hour of paid implementer calls is
10819    // not thrown away over cache contention — but every write-allowed seat
10820    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10821    // below), the same fallback a read-only seat always gets, rather than
10822    // building into a directory this run was never granted. The identity
10823    // record is still invalidated below either way, so the next tracked
10824    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10825    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10826    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10827    // never borrowed from a job's agent budget (and never charged to it either,
10828    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10829    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10830    let cache_guard = if let Some(cache_dir) = cache {
10831        if jobs_had_a_writer {
10832            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10833            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10834            match acquire_cache_lease(
10835                state,
10836                cache_dir,
10837                &owner,
10838                resource_limit,
10839                node,
10840                node,
10841                &cancel,
10842            )
10843            .await
10844            {
10845                Ok(Some(g)) => Some(g),
10846                // Parking: leave without spawning anything. The run parks at
10847                // the next boundary and the seats are asked afresh on resume.
10848                Ok(None) => {
10849                    if let Some(block) = &mut state.resource_block {
10850                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10851                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10852                        };
10853                    }
10854                    return block_wave(
10855                        jobs,
10856                        state,
10857                        node,
10858                        attempt,
10859                        "parked while waiting for the shared build cache",
10860                    );
10861                }
10862                // An unreadable lease is not contention: keep the old
10863                // fallback (no shared cache for this wave's seats).
10864                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10865                Err(e) => {
10866                    // Writing seats may not build into a cache this run does
10867                    // not hold, and an agent that cannot be given one is not
10868                    // asked at all: the run stops resumably instead.
10869                    let why = format!("{e:#}");
10870                    state.event(
10871                        "resource",
10872                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10873                    );
10874                    return block_wave(jobs, state, node, attempt, &why);
10875                }
10876            }
10877        } else {
10878            None
10879        }
10880    } else {
10881        None
10882    };
10883    // Every job keeps its whole budget: the wait above was not the agent's.
10884    // Restart the seats' clocks so "started" means "being spawned now".
10885    state.seats_spawning(node, attempt);
10886    if let Err(e) = state.save() {
10887        tracing::warn!("could not persist the start of a wave: {e:#}");
10888    }
10889    // Queueing behind this wave's own seats for a parallelism slot is normal
10890    // (each of them is bounded by its own agent timeout); only a wait beyond
10891    // all of them plus the resource limit means the slots never came back.
10892    let slot_limit = resource_limit
10893        + jobs
10894            .iter()
10895            .map(|j| j.timeout)
10896            .max()
10897            .unwrap_or_default()
10898            .saturating_mul(jobs.len() as u32);
10899    let mut set = tokio::task::JoinSet::new();
10900    // Seats announce themselves here the moment they hold a parallelism
10901    // permit, i.e. just before their CLI is spawned.
10902    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10903    let overlay = prompts.overlay(node);
10904    let repository_wait = repository_wait(state, pauses);
10905    for (i, mut job) in jobs.into_iter().enumerate() {
10906        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10907        if cache.is_some() {
10908            job.prompt.push('\n');
10909            job.prompt
10910                .push_str(&prompt::build_cache_note(node, job.allow_write));
10911        }
10912        let sem = Arc::clone(&sem);
10913        let run = run.to_owned();
10914        let node = node.to_owned();
10915        // Only implementers were told about the task's attachments, so only
10916        // their seats get the directory widened for reading.
10917        let attachments = if node == "implement" {
10918            state.attachments.clone()
10919        } else {
10920            Vec::new()
10921        };
10922        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10923        // `prompt::build_cache_note`'s doc for why setting it anyway is
10924        // exactly how a sandboxed reviewer's write refusal got reported as a
10925        // defect in the patch, not a property of its own seat. And a
10926        // write-allowed one is handed it only when the lease above was
10927        // actually acquired: a wave that could not get it (`cache_guard` is
10928        // `None`, see its own comment) must not send seats to build into a
10929        // directory this run does not hold - that is the exact concurrent,
10930        // unmanaged-write race this module exists to prevent, not something
10931        // "proceeding anyway" is allowed to reintroduce.
10932        let cache = cache
10933            .filter(|_| job.allow_write && cache_guard.is_some())
10934            .map(Path::to_path_buf);
10935        let spawned_tx = spawned_tx.clone();
10936        let repository_wait = repository_wait.clone();
10937        let coordination = state.config.coordination.clone();
10938        let fetched_base = state.fetched_base.clone();
10939        set.spawn(async move {
10940            // A wait for a parallelism slot is scheduling, not the agent: it
10941            // is bounded by its own limit and ends as a resource block, never
10942            // as a timeout of the agent that was not started.
10943            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10944                Ok(p) => p,
10945                Err(_) => {
10946                    return (
10947                        i,
10948                        job.seat,
10949                        AgentOutcome::ResourceBlocked(format!(
10950                            "no parallel slot became free within {}s",
10951                            slot_limit.as_secs()
10952                        )),
10953                        false,
10954                    );
10955                }
10956            };
10957            spawned_tx.send(job.seat.key.clone()).ok();
10958            let mut seat = job.seat;
10959            let out = repository_wait
10960                .scope(agent::invoke(
10961                    &job.spec,
10962                    &mut seat,
10963                    &Invocation {
10964                        coordination: coordination.as_ref(),
10965                        fetched_base: fetched_base.as_ref(),
10966                        cwd: &job.cwd,
10967                        prompt: &job.prompt,
10968                        timeout: job.timeout,
10969                        allow_write: job.allow_write,
10970                        sessions: job.sessions,
10971                        artifacts: &job.artifacts,
10972                        stem: &job.stem,
10973                        run: &run,
10974                        node: &node,
10975                        cache_dir: cache.as_deref(),
10976                        attachments: &attachments,
10977                        writable: &[],
10978                    },
10979                ))
10980                .await;
10981            let interrupted = out
10982                .as_ref()
10983                .err()
10984                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10985            let out = match out {
10986                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10987                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10988                // Billed work the CLI failed to hand over is not an ordinary
10989                // failure, but its text is the CLI's raw error JSON, not an
10990                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10991                // read it as one by forgetting to check. `usable()` is always
10992                // false here (dropped implies an empty response), so this has
10993                // to be checked before the catch-all `Failed` below or the
10994                // one shape this exists for is lost with the rest.
10995                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10996                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10997                Ok(o) => AgentOutcome::Failed(format!(
10998                    "process exited with no usable output ({})",
10999                    crate::notices::exit_status(o.exit_code)
11000                )),
11001                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
11002                    AgentOutcome::ResourceBlocked(e.to_string())
11003                }
11004                Err(e) => AgentOutcome::Failed(e.to_string()),
11005            };
11006            (i, seat, out, interrupted)
11007        });
11008    }
11009    drop(spawned_tx);
11010    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
11011    loop {
11012        let joined = tokio::select! {
11013            Some(key) = spawned_rx.recv() => {
11014                // The clock of this seat starts now, not when it queued.
11015                state.seat_spawned(&key);
11016                if let Err(e) = state.save() {
11017                    tracing::warn!("could not persist a seat's start: {e:#}");
11018                }
11019                continue;
11020            }
11021            joined = set.join_next() => joined,
11022        };
11023        let Some(joined) = joined else {
11024            break;
11025        };
11026        let (i, seat, out, interrupted) = match joined {
11027            Ok(v) => v,
11028            // No seat to clear: a panicked task never reported which one it
11029            // was. The defensive sweep below this loop is what stops that
11030            // seat's `active` entry from surviving forever.
11031            Err(e) => {
11032                tracing::error!("agent task panicked: {e}");
11033                continue;
11034            }
11035        };
11036        state.seat_finished(&seat.key);
11037        if interrupted
11038            && node == "implement"
11039            && let Some(label) = state
11040                .candidates
11041                .iter()
11042                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11043                .map(|candidate| candidate.label)
11044        {
11045            state.candidate_preservation.insert(
11046                label,
11047                "interrupted implementation awaiting persistence".into(),
11048            );
11049            state.interrupted_implementations.insert(label);
11050        }
11051        if let AgentOutcome::ResourceBlocked(why) = &out
11052            && state.resource_block.is_none()
11053        {
11054            state.resource_block = Some(crate::run::ResourceBlock {
11055                cause: crate::run::ResourceBlockCause::Other,
11056                node: node.to_owned(),
11057                kind: if why.contains("disk pressure") {
11058                    "disk pressure"
11059                } else if why.contains("disk admission") {
11060                    "disk admission"
11061                } else if why.contains(crate::incremental::BLOCKED) {
11062                    "incremental cache admission"
11063                } else if why.contains("repository coordination") {
11064                    "repository coordination"
11065                } else {
11066                    "parallel slot"
11067                }
11068                .to_owned(),
11069                detail: why.clone(),
11070                at: Timestamp::now(),
11071            });
11072            let activity = if interrupted {
11073                "agent execution interrupted"
11074            } else {
11075                "no agent was started"
11076            };
11077            state.event("resource", format!("{node}: {activity} - {why}"));
11078        }
11079        record_jobs(state, node, round, &seat.key, &out);
11080        if let Err(e) = state.save() {
11081            tracing::warn!("could not persist a seat's completion: {e:#}");
11082        }
11083        if collected.len() <= i {
11084            collected.resize_with(i + 1, || None);
11085        }
11086        collected[i] = Some((i, seat, out));
11087    }
11088    // Belt-and-braces for the panic branch above: every seat this exact batch
11089    // started shares this `(node, attempt)` pair, and every seat that finished
11090    // normally already cleared itself, so anything left tagged with it here
11091    // can only be a panicked task's leftover. Cleared unconditionally rather
11092    // than left to read as still answering forever.
11093    if state
11094        .active
11095        .values()
11096        .any(|a| a.node == node && a.attempt == attempt)
11097    {
11098        state
11099            .active
11100            .retain(|_, a| !(a.node == node && a.attempt == attempt));
11101        if let Err(e) = state.save() {
11102            tracing::warn!("could not persist the end of a wave: {e:#}");
11103        }
11104    }
11105    // Whether or not the lease above was actually held, several worktrees
11106    // may just have built into the cache with nothing here able to name one
11107    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
11108    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
11109    // clean it might not have strictly needed; trusting a stale match would
11110    // cost it a wrong answer.
11111    if let Some(cache_dir) = cache
11112        && jobs_had_a_writer
11113    {
11114        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11115    }
11116    if let Some(guard) = cache_guard {
11117        guard.release();
11118    }
11119    collected.into_iter().flatten().collect()
11120}
11121
11122/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
11123/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
11124/// this is data collection, not the fix-specific completion contract in
11125/// [`Runner::continue_fix_report`], and applies regardless of which node
11126/// asked.
11127///
11128/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11129/// evidence from; `Failed` does not, and correctly contributes nothing — a
11130/// timeout or crash is not itself evidence about a command the seat may have
11131/// started.
11132fn record_jobs(
11133    state: &mut RunState,
11134    node: &str,
11135    round: Option<usize>,
11136    seat: &str,
11137    out: &AgentOutcome,
11138) {
11139    let commands: &[agent::CommandEvidence] = match out {
11140        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11141        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11142    };
11143    let checked_at = Timestamp::now();
11144    for c in commands {
11145        state.jobs.push(JobRecord {
11146            node: node.to_owned(),
11147            round,
11148            seat: seat.to_owned(),
11149            id: c.id.clone(),
11150            description: c.description.clone(),
11151            checked_at,
11152            status: match c.exit_code {
11153                Some(0) => JobStatus::Completed,
11154                Some(_) => JobStatus::Failed,
11155                None => JobStatus::Unknown,
11156            },
11157            exit_code: c.exit_code,
11158            result_summary: c.result_summary.clone(),
11159            source: c.source.clone(),
11160        });
11161    }
11162}
11163
11164/// Is a review round clean, given how many reviewer seats answered against
11165/// how many the round expected?
11166///
11167/// A seat that never answered (timeout, crash, unparsable output) is not a
11168/// seat that read the patch and found nothing — treating it as such is
11169/// exactly the bug this function exists to close. Under the default `block`
11170/// policy a missing seat can never be clean; `warn` still requires the seats
11171/// that *did* answer to have found nothing blocking and verification to be
11172/// green.
11173///
11174/// `quota_missing` narrows that `block` default for exactly one cause of
11175/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11176/// a session limit lifts by the very next round buys nothing — the seat is
11177/// asked again with the same quota — so once every missing seat is accounted
11178/// for by a quota loss (and at least one seat *did* answer, so a decision has
11179/// something to rest on) the round is decided on the panel that could answer,
11180/// same as `warn` would. A panel that lost every seat to quota is not
11181/// decided here: `answered == 0` falls through to the existing `block`
11182/// fallback so a fully collapsed panel still waits rather than landing on no
11183/// review at all.
11184fn round_is_clean(
11185    blocking: usize,
11186    e2e_ok: bool,
11187    answered: usize,
11188    expected: usize,
11189    quota_missing: usize,
11190    policy: IncompleteReviewPolicy,
11191) -> bool {
11192    if blocking != 0 || !e2e_ok {
11193        return false;
11194    }
11195    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11196        return true;
11197    }
11198    answered > 0 && expected - answered <= quota_missing
11199}
11200
11201/// The review loop's own conclusion, derived entirely from its persisted
11202/// round records and the round budget that produced them — never from
11203/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11204/// recomputes the identical answer regardless of what an earlier node in the
11205/// same walk, or a previous walk, did to `status`.
11206///
11207/// `None` while more rounds remain to try, including when review never ran
11208/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11209/// gone clean, or the budget is spent, or the tree has stopped moving (see
11210/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11211///
11212/// - An incomplete panel that raised nothing is missing input, not a
11213///   verified tree — never a hand-off candidate, whatever verification said
11214///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11215/// - Otherwise, green e2e on the last round hands off (see
11216///   [`Runner::stop_reviewing`]); red e2e blocks.
11217///
11218/// A last round whose own verification is still `ResourceBlocked` — magi
11219/// itself never got a command to run, not evidence the patch is broken —
11220/// is neither: this returns `None` for it too, the same as "more rounds
11221/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11222/// handling of that shape) instead of this cheap recomputation guessing a
11223/// verdict a real attempt never produced.
11224pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11225    if max_rounds == 0
11226        || reviews.last().is_some_and(|r| {
11227            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11228        })
11229    {
11230        return Some(RunStatus::Gating);
11231    }
11232    let last = reviews.last()?;
11233    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11234    if reviews.len() < max_rounds && !stagnant {
11235        return None;
11236    }
11237    if last.incomplete() && last.blocking == 0 {
11238        return Some(RunStatus::Blocked);
11239    }
11240    if last.e2e_status() == E2eStatus::ResourceBlocked
11241        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11242    {
11243        return None;
11244    }
11245    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11246        RunStatus::Gating
11247    } else {
11248        RunStatus::Blocked
11249    })
11250}
11251
11252/// How long a re-ask may take, given the budget the first attempt had.
11253///
11254/// A `nudged` retry is a request to restate an answer the seat has already
11255/// worked out: it carries no new work, so it does not deserve the original
11256/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11257/// 133 seconds while a third sat for over ten minutes on a resumed session
11258/// holding 410 KB of prior output - and because the retry had inherited the
11259/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11260/// judging round whose other seats were long finished.
11261///
11262/// A quarter of the budget, with a floor so that a deliberately short timeout
11263/// does not collapse to nothing. A retry that re-sends the whole prompt
11264/// (because the seat kept no context) is the original job again, and keeps the
11265/// original budget.
11266pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11267    if nudged {
11268        (full / 4).max(Duration::from_secs(120)).min(full)
11269    } else {
11270        full
11271    }
11272}
11273
11274/// Run a wave and parse each reply, re-asking the seats whose reply was
11275/// unusable.
11276///
11277/// The re-ask is a nudge rather than the whole prompt again when the seat still
11278/// holds its conversation, which is the difference between a cheap retry and
11279/// paying for the entire candidate set twice.
11280///
11281/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11282/// successor in `roster` is handed to it instead of being re-asked: the
11283/// handover is the retry. A seat with no successor left (a single-agent
11284/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11285/// the asks to one seat in one node number at most
11286/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11287/// is asked once (a dropped stream is nudged first), and only the last agent
11288/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11289/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11290/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11291/// other failure stops the chain when the previous agent failed the same way
11292/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11293/// [`has_context`] is false and the job's own full prompt and full budget are
11294/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11295/// intermediate ones are not losses) and is returned as a failure like any
11296/// other absent seat — the caller decides whether the panel still has a
11297/// quorum. An empty `roster` disables handover: failures are nudged as they
11298/// always were, and a quota is simply lost. A reply that fails to parse or
11299/// validate is the prompt's doing and is only ever nudged, never handed over.
11300///
11301/// The returned [`SeatState`] names the agent that answered (or tried last).
11302#[allow(clippy::too_many_arguments)]
11303async fn ask_json_wave<T>(
11304    jobs: Vec<SeatJob>,
11305    sem: Arc<Semaphore>,
11306    retries: usize,
11307    roster: &[AgentSpec],
11308    ctx: &WaveCtx<'_>,
11309    losses: &mut Vec<QuotaLoss>,
11310    state: &mut RunState,
11311    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11312) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11313where
11314    T: serde::de::DeserializeOwned + Send + 'static,
11315{
11316    ask_wave_with(
11317        jobs,
11318        sem,
11319        retries,
11320        roster,
11321        ctx,
11322        losses,
11323        state,
11324        &|text: &str| {
11325            let v = verdict::extract_json::<T>(text)?;
11326            validate(&v)?;
11327            Ok(v)
11328        },
11329    )
11330    .await
11331}
11332
11333/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11334/// a node whose answer is prose (deliberation) shares the same handover,
11335/// failure classification, quota bookkeeping and bounds instead of a copy.
11336///
11337/// A seat handed to another roster agent is sent the job's `handover` prompt
11338/// (when it has one) rather than `prompt`: the new agent has no session, so a
11339/// resume-style prompt would be incomplete. That holds for the handover ask
11340/// and for every nudge to that agent whose `has_context` is false.
11341#[allow(clippy::too_many_arguments)]
11342async fn ask_wave_with<T>(
11343    jobs: Vec<SeatJob>,
11344    sem: Arc<Semaphore>,
11345    retries: usize,
11346    roster: &[AgentSpec],
11347    ctx: &WaveCtx<'_>,
11348    losses: &mut Vec<QuotaLoss>,
11349    state: &mut RunState,
11350    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11351) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11352where
11353    T: Send + 'static,
11354{
11355    let n = jobs.len();
11356    let originals: Vec<SeatJob> = jobs;
11357    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11358    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11359    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11360    // answer, N once it has gone through N nudges. Read back once this
11361    // returns, so a caller building a history record (`ReviewRecord`) can
11362    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11363    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11364    // field's own doc.
11365    let mut nudges: Vec<usize> = vec![0; n];
11366    // The agent now occupying each seat, the ids it has already been through,
11367    // where in the roster the walk began, the class of the last failure, and
11368    // the stem word of a handover not yet asked (full prompt, full budget).
11369    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11370    let mut tried: Vec<BTreeSet<String>> = specs
11371        .iter()
11372        .map(|s| BTreeSet::from([s.id.clone()]))
11373        .collect();
11374    let starts: Vec<usize> = specs
11375        .iter()
11376        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11377        .collect();
11378    let carry = ctx.carry_seats && !roster.is_empty();
11379    // Carried across rounds: ids that failed the seat earlier, and how the
11380    // last failure went (so a repeat of it is not handed over again).
11381    let carried: Vec<BTreeSet<String>> = originals
11382        .iter()
11383        .map(|j| {
11384            state
11385                .seat_history
11386                .get(&j.seat.key)
11387                .filter(|_| carry)
11388                .map(|h| h.failed.clone())
11389                .unwrap_or_default()
11390        })
11391        .collect();
11392    let mut prev: Vec<Option<FailClass>> = originals
11393        .iter()
11394        .map(|j| {
11395            state
11396                .seat_history
11397                .get(&j.seat.key)
11398                .filter(|_| carry)
11399                .and_then(|h| h.last_fail.clone())
11400        })
11401        .collect();
11402    let next_agent =
11403        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11404            let others: BTreeSet<String> = specs
11405                .iter()
11406                .enumerate()
11407                .filter(|(j, _)| *j != i)
11408                .map(|(_, s)| s.id.clone())
11409                .collect();
11410            pick_successor(
11411                roster,
11412                starts[i],
11413                tried,
11414                carry.then(|| &carried[i]),
11415                &others,
11416            )
11417            .cloned()
11418        };
11419    let mut fresh: Vec<Option<String>> = vec![None; n];
11420    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11421    let mut pending: Vec<usize> = (0..n).collect();
11422
11423    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11424    // asks: an agent with a successor is asked once and handed over, and only
11425    // a seat with no successor spends `retries` nudges on the same agent. This
11426    // only guarantees the loop's own termination whatever those say.
11427    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11428    for round in 0..max_rounds {
11429        if pending.is_empty() {
11430            break;
11431        }
11432        let mut batch = Vec::with_capacity(pending.len());
11433        let mut renudged: Vec<&str> = Vec::new();
11434        for &i in &pending {
11435            let src = &originals[i];
11436            // A seat now held by another agent than the job named has no
11437            // session of its own: it gets the full-context prompt whenever
11438            // it is asked in full (the handover ask, a nudge it cannot
11439            // resume).
11440            let full: &str = match &src.handover {
11441                Some(h) if specs[i].id != src.spec.id => h,
11442                _ => &src.prompt,
11443            };
11444            // The prompt and the budget are one decision: a nudge restates
11445            // finished work, a re-sent prompt redoes it.
11446            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11447                (
11448                    full.to_owned(),
11449                    src.timeout,
11450                    format!("{}-{word}-{}", src.stem, specs[i].id),
11451                )
11452            } else if nudges[i] == 0 {
11453                (full.to_owned(), src.timeout, src.stem.clone())
11454            } else {
11455                renudged.push(src.seat.key.as_str());
11456                let why = done[i]
11457                    .as_ref()
11458                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11459                    .unwrap_or_else(|| "no parsable answer".to_owned());
11460                let nudge = prompt::nudge(&why);
11461                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11462                let prompt = if nudged {
11463                    nudge
11464                } else {
11465                    format!("{full}\n\n---\n\n{nudge}")
11466                };
11467                (
11468                    prompt,
11469                    retry_budget(src.timeout, nudged),
11470                    format!("{}-retry{}", src.stem, nudges[i]),
11471                )
11472            };
11473            batch.push(SeatJob {
11474                spec: specs[i].clone(),
11475                seat: seats[i].clone(),
11476                cwd: src.cwd.clone(),
11477                prompt,
11478                timeout,
11479                allow_write: src.allow_write,
11480                sessions: src.sessions,
11481                artifacts: src.artifacts.clone(),
11482                stem,
11483                handover: None,
11484            });
11485        }
11486
11487        if !renudged.is_empty() {
11488            state.event(
11489                ctx.node,
11490                format!("retry {round}: re-asking {}", renudged.join(", ")),
11491            );
11492        }
11493        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11494        let mut still = Vec::new();
11495        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11496            seats[i] = seat;
11497            // No agent was asked: nothing to hand over, nudge, remember or
11498            // charge. The caller sees `resource_block` and stops the node.
11499            if let AgentOutcome::ResourceBlocked(why) = &out {
11500                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11501                continue;
11502            }
11503            let class = FailClass::of(&out);
11504            // A dropped stream is nudged first (the conversation is still
11505            // there to pick up); only a seat whose nudges are spent hands over.
11506            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11507                && nudges[i] < retries
11508                && !roster.is_empty();
11509            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11510                let next = should_hand_over(prev[i].as_ref(), &cur)
11511                    .then(|| next_agent(i, &tried[i], &specs))
11512                    .flatten();
11513                if carry {
11514                    let h = state
11515                        .seat_history
11516                        .entry(originals[i].seat.key.clone())
11517                        .or_default();
11518                    h.failed.insert(specs[i].id.clone());
11519                    h.last_fail = Some(cur.clone());
11520                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11521                        h.last_ok = None;
11522                    }
11523                    // Saved before the next agent is asked, so a restart in
11524                    // between does not forget who failed.
11525                    if let Err(e) = state.save() {
11526                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11527                    }
11528                }
11529                if let Some(next) = next {
11530                    record_handover(
11531                        state,
11532                        ctx.node,
11533                        &originals[i].seat.key,
11534                        &specs[i].id,
11535                        &next.id,
11536                        &cur,
11537                        &fail_reason(&out),
11538                    );
11539                    tried[i].insert(next.id.clone());
11540                    prev[i] = Some(cur.clone());
11541                    seats[i] =
11542                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11543                    specs[i] = next;
11544                    fresh[i] = Some(cur.stem_word().to_owned());
11545                    nudges[i] = 0;
11546                    done[i] = Some(Err(anyhow::anyhow!(
11547                        "handed over after: {}",
11548                        fail_reason(&out)
11549                    )));
11550                    still.push(i);
11551                    continue;
11552                }
11553            }
11554            if carry
11555                && !nudge_first
11556                && let Some(cur) = class.clone()
11557            {
11558                // The chain ended here (no successor, or a repeated failure).
11559                let h = state
11560                    .seat_history
11561                    .entry(originals[i].seat.key.clone())
11562                    .or_default();
11563                h.failed.insert(specs[i].id.clone());
11564                h.last_fail = Some(cur);
11565            }
11566            let parsed = match out {
11567                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11568                AgentOutcome::Quota(o) => {
11569                    last_quota[i] = o.quota.clone();
11570                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11571                }
11572                // Not a parseable answer: the nudge loop re-asks it, which is
11573                // exactly what a dropped stream needs. Just don't hand its raw
11574                // error JSON to `extract_json`.
11575                AgentOutcome::Dropped(o) => {
11576                    let why = o
11577                        .dropped
11578                        .as_ref()
11579                        .map(|d| d.why.as_str())
11580                        .unwrap_or("the CLI ended the stream without delivering its answer");
11581                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11582                }
11583                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11584                    Err(anyhow::anyhow!(e))
11585                }
11586            };
11587            let quota = class == Some(FailClass::Quota);
11588            let failed = parsed.is_err();
11589            if !failed && matches!(ctx.node, "judge" | "vote") {
11590                // A validated answer restores this seat's quorum presence,
11591                // including on the normal path after an all-seat block.
11592                state
11593                    .quota
11594                    .retain(|loss| loss.seat != originals[i].seat.key);
11595            }
11596            done[i] = Some(parsed);
11597            if carry && !failed {
11598                let h = state
11599                    .seat_history
11600                    .entry(originals[i].seat.key.clone())
11601                    .or_default();
11602                h.failed.remove(&specs[i].id);
11603                h.last_ok = Some(specs[i].id.clone());
11604                h.last_fail = None;
11605            }
11606            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11607            // fail the same way; and never re-ask a seat that already parsed.
11608            // A failed agent that still has a successor is not re-asked
11609            // either: the handover was its remedy and has just been refused
11610            // (the chain stops on a repeated failure class). A seat with no
11611            // successor left (a single-agent roster, the roster's tail, or an
11612            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11613            let agent_failure = class.is_some()
11614                && !nudge_first
11615                && !roster.is_empty()
11616                && next_agent(i, &tried[i], &specs).is_some();
11617            if failed && !quota && !agent_failure && nudges[i] < retries {
11618                nudges[i] += 1;
11619                still.push(i);
11620            }
11621        }
11622        pending = still;
11623    }
11624
11625    // One loss per seat whose chain ended on a quota: the intermediate ones
11626    // were absorbed by a handover and are not losses.
11627    for (i, q) in last_quota.into_iter().enumerate() {
11628        if let Some(quota) = q {
11629            losses.push(QuotaLoss::from_report(
11630                originals[i].seat.key.clone(),
11631                ctx.node,
11632                Some(&quota),
11633            ));
11634        }
11635    }
11636
11637    // A stalled quorum already has its own recovery checkpoint. Refresh
11638    // its missing seats before deciding whether it can resume.
11639    if state.status != RunStatus::Stalled
11640        && ctx.node != "deliberate"
11641        && n > 0
11642        && losses.len() == n
11643        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11644    {
11645        block_machine_outage(state, ctx.node, losses);
11646        state.quota.append(losses);
11647    }
11648
11649    seats
11650        .into_iter()
11651        .zip(done)
11652        .zip(nudges)
11653        .map(|((seat, res), attempts)| {
11654            (
11655                seat,
11656                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11657                attempts,
11658            )
11659        })
11660        .collect()
11661}
11662
11663/// A node with no available agent has no task verdict. Preserve the node
11664/// checkpoint and record why a subsequent explicit resume is needed.
11665pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11666    let mut auth: Vec<_> = losses
11667        .iter()
11668        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11669        .map(|loss| loss.agent.clone())
11670        .collect();
11671    auth.sort();
11672    auth.dedup();
11673    let (kind, detail) = if auth.is_empty() {
11674        (
11675            "agent usage limit",
11676            "All seats reached their agent usage limits; resume after usage is available"
11677                .to_owned(),
11678        )
11679    } else {
11680        (
11681            "agent authentication",
11682            format!(
11683                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11684                auth.join(", ")
11685            ),
11686        )
11687    };
11688    state.resource_block = Some(crate::run::ResourceBlock {
11689        node: node.to_owned(),
11690        kind: kind.to_owned(),
11691        cause: crate::run::ResourceBlockCause::Other,
11692        detail: detail.clone(),
11693        at: Timestamp::now(),
11694    });
11695    state.event(node, detail);
11696}
11697
11698/// Did the wave end without asking any agent (see [`block_wave`])?
11699fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11700    results
11701        .iter()
11702        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11703}
11704
11705/// Settle a wave that never spawned anything: every job answers
11706/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11707/// else about the seat (session, history, quota) is touched.
11708fn block_wave(
11709    jobs: Vec<SeatJob>,
11710    state: &mut RunState,
11711    node: &str,
11712    attempt: usize,
11713    why: &str,
11714) -> Vec<(usize, SeatState, AgentOutcome)> {
11715    state.resource_wait = None;
11716    state
11717        .active
11718        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11719    if let Err(e) = state.save() {
11720        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11721    }
11722    jobs.into_iter()
11723        .enumerate()
11724        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11725        .collect()
11726}
11727
11728/// Acquire the shared build cache's lease, waiting out contention within
11729/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11730/// unbounded wait is never acceptable).
11731///
11732/// A first, non-blocking check happens before ever waiting; if it finds the
11733/// lease busy, that fact is logged as a `verify` event *and* flushed with
11734/// [`RunState::save`] immediately — not only once the wait finally succeeds
11735/// or gives up — so a `magi show` run by a different process while this one
11736/// is still waiting reads a `run.json` that says so, rather than whatever it
11737/// looked like before the wait started. The same applies to the terminal
11738/// failure: logged and saved before this returns `Err`, so a caller that
11739/// could not get the lease at all still leaves a legible record of why.
11740async fn acquire_cache_lease(
11741    state: &mut RunState,
11742    cache_dir: &Path,
11743    owner: &crate::cache::Owner,
11744    budget: Duration,
11745    node: &str,
11746    context: &str,
11747    cancel: &(dyn Fn() -> bool + Sync),
11748) -> Result<Option<crate::cache::Guard>> {
11749    let home = crate::run::home();
11750    let started = Instant::now();
11751    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11752        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11753        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11754        Err(e) => {
11755            state.event(
11756                "verify",
11757                format!("{context}: could not check the shared build cache: {e:#}"),
11758            );
11759            if let Err(e2) = state.save() {
11760                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11761            }
11762            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11763        }
11764    };
11765    state.event(
11766        "verify",
11767        format!(
11768            "{context}: waiting for the shared build cache at {} ({})",
11769            cache_dir.display(),
11770            busy.describe()
11771        ),
11772    );
11773    // Seats of this node are waiting, not running: say so, so the countdown a
11774    // reader sees only ever measures an agent that has been spawned.
11775    state.resource_wait = Some(crate::run::ResourceWait {
11776        node: context.to_owned(),
11777        kind: "shared build cache".to_owned(),
11778        holder: busy.describe(),
11779        since: Timestamp::now(),
11780        limit_secs: budget.as_secs(),
11781    });
11782    if let Err(e) = state.save() {
11783        tracing::warn!("could not persist a cache wait: {e:#}");
11784    }
11785    let remaining = budget.saturating_sub(started.elapsed());
11786    let waited = crate::cache::wait_for_unless(
11787        &home,
11788        cache_dir,
11789        owner,
11790        remaining,
11791        Duration::from_secs(5),
11792        cancel,
11793    )
11794    .await;
11795    state.resource_wait = None;
11796    let cause = match &waited {
11797        Ok(Some(_)) => None,
11798        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11799        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11800            Some(crate::run::ResourceBlockCause::Timeout {
11801                waited_secs: started.elapsed().as_secs(),
11802                limit_secs: budget.as_secs(),
11803            })
11804        }
11805        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11806    };
11807    if let Some(cause) = cause {
11808        let detail = match &waited {
11809            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11810            Err(error) => format!("{error:#}"),
11811        };
11812        state.resource_block = Some(crate::run::ResourceBlock {
11813            cause,
11814            node: node.to_owned(),
11815            kind: "shared build cache".into(),
11816            detail,
11817            at: Timestamp::now(),
11818        });
11819        state.event(
11820            "resource",
11821            state.resource_block.as_ref().unwrap().description(),
11822        );
11823    }
11824    if let Err(error) = state.save() {
11825        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11826    }
11827    waited
11828}
11829
11830/// The lease state could not be read at all (unreadable lease file, a
11831/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11832/// known to be contended, so a wave degrades to running without a shared cache
11833/// instead of stopping on a hold that [`crate::triage`] could never release.
11834#[derive(Debug)]
11835struct CacheCheckError(String);
11836
11837impl std::fmt::Display for CacheCheckError {
11838    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11839        f.write_str(&self.0)
11840    }
11841}
11842
11843impl std::error::Error for CacheCheckError {}
11844
11845/// Keep a specific acquisition outcome when the caller adds command evidence.
11846fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11847    state
11848        .resource_block
11849        .get_or_insert_with(|| crate::run::ResourceBlock {
11850            cause: crate::run::ResourceBlockCause::Other,
11851            node: node.to_owned(),
11852            kind: "shared build cache".into(),
11853            detail: format!("{error:#}"),
11854            at: Timestamp::now(),
11855        });
11856    if let Err(error) = state.save() {
11857        tracing::warn!("could not persist cache resource block: {error:#}");
11858    }
11859}
11860
11861/// Run `body` — a verify command batch — while holding the shared build
11862/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11863/// never interleave with another borrower's build against the same
11864/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11865/// timeout, or a human's own `magi review`. See the `cache` module doc for
11866/// why this matters more than Cargo's own per-target locking covers — two
11867/// *different* worktrees building the same package name/version into one
11868/// cache directory is a staleness bug, not a lock contention one.
11869///
11870/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11871/// agent waves do. Repository scheduling also has its own bounded,
11872/// cancellable resource-wait policy. Contention must not consume the
11873/// verification budget: `body` receives the full `budget` after both leases
11874/// are acquired and the cache is fresh. Without a shared cache, only
11875/// repository coordination is acquired before handing over the full budget.
11876///
11877/// A lease that cannot be acquired within the resource wait is reported as a single
11878/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11879/// verification — the same shape a spawn failure already takes in
11880/// [`run_commands`], so a caller need not special-case it.
11881#[allow(clippy::too_many_arguments)]
11882async fn with_cache_lease<'s, F, Fut>(
11883    state: &'s mut RunState,
11884    cache_dir: Option<&Path>,
11885    node: &str,
11886    seat: &str,
11887    worktree: &Path,
11888    head: &str,
11889    budget: Duration,
11890    context: &str,
11891    wait: &crate::repo_lock::WaitPolicy,
11892    body: F,
11893) -> Result<(Vec<CommandOutcome>, bool)>
11894where
11895    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11896    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11897{
11898    // Match editing waves: cache first, repository second. In particular,
11899    // pre_gate must never hold the repository while waiting for a wave's cache.
11900    let started = Instant::now();
11901    let home = crate::run::home();
11902    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11903    let guard = if let Some(cache_dir) = cache_dir {
11904        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11905        match acquire_cache_lease(
11906            state,
11907            cache_dir,
11908            &owner,
11909            resource_limit,
11910            node,
11911            context,
11912            &|| wait.cancelled(),
11913        )
11914        .await
11915        .and_then(|g| {
11916            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11917        }) {
11918            Ok(g) => Some(g),
11919            Err(e) => {
11920                record_cache_error(state, node, &e);
11921                return Ok((
11922                    vec![CommandOutcome {
11923                        command: "(waiting for the shared build cache)".into(),
11924                        code: None,
11925                        output_tail: e.to_string(),
11926                        duration_ms: started.elapsed().as_millis() as u64,
11927                        resource_blocked: true,
11928                    }],
11929                    false,
11930                ));
11931            }
11932        }
11933    } else {
11934        None
11935    };
11936    let _coordination = coordinate_operation(state, worktree, wait).await?;
11937    if let Some(cache_dir) = cache_dir {
11938        let identity = crate::cache::Identity::new(worktree, head);
11939        if let Err(e) = crate::cache::ensure_fresh(
11940            &home,
11941            cache_dir,
11942            &identity,
11943            &state.config.verify.cargo_manifests(),
11944        ) {
11945            record_cache_error(state, node, &e);
11946            state.event(
11947                "verify",
11948                format!(
11949                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11950                    worktree.display(),
11951                    short(head)
11952                ),
11953            );
11954            if let Some(guard) = guard {
11955                guard.release();
11956            }
11957            return Ok((
11958                vec![CommandOutcome {
11959                    command: "(confirming the shared build cache is fresh)".into(),
11960                    code: None,
11961                    output_tail: e.to_string(),
11962                    duration_ms: started.elapsed().as_millis() as u64,
11963                    resource_blocked: true,
11964                }],
11965                false,
11966            ));
11967        }
11968    }
11969    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11970    if !timed_out_pids.is_empty() {
11971        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11972    }
11973    // Drop repository coordination before releasing the cache, reversing the
11974    // acquisition order. A cancelled/error future also drops both guards.
11975    drop(_coordination);
11976    if let Some(guard) = guard {
11977        guard.release();
11978    }
11979    Ok((outcomes, retried))
11980}
11981
11982/// Poll `pids` — commands [`run_commands`] reports as still running when its
11983/// own timeout elapsed — until every one is confirmed gone, or
11984/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11985///
11986/// Real confirmation where confirmation is possible, not a substitute for
11987/// full process-tree observation: a grandchild the timed-out process spawned
11988/// and that survives independently of it is invisible to a pid check the
11989/// same way it always was, and continuing to observe and collect *that*
11990/// stays a different piece of work with its own owner. This only narrows a
11991/// fixed blind wait into an actual check of the pids this process does know
11992/// about.
11993async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11994    wait_for_pids_with(
11995        pids,
11996        crate::proc::pid_alive,
11997        LEASE_RELEASE_POLL,
11998        LEASE_RELEASE_MAX_WAIT,
11999    )
12000    .await;
12001}
12002
12003/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
12004/// interval and ceiling supplied by the caller, so the polling *logic* -
12005/// returns as soon as every pid reports dead, gives up at the ceiling
12006/// otherwise - is testable on millisecond durations without asking the real
12007/// OS about a pid at all.
12008async fn wait_for_pids_with<F: Fn(u32) -> bool>(
12009    pids: &[u32],
12010    alive: F,
12011    poll: Duration,
12012    max_wait: Duration,
12013) {
12014    let deadline = Instant::now() + max_wait;
12015    loop {
12016        if pids.iter().all(|&pid| !alive(pid)) {
12017            return;
12018        }
12019        if Instant::now() >= deadline {
12020            return;
12021        }
12022        tokio::time::sleep(poll).await;
12023    }
12024}
12025
12026/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
12027/// admission that it could not even get a verify command to run, as opposed
12028/// to evidence the command actually produced? A caller that would otherwise
12029/// read a resource-blocked outcome as a red command must check this first:
12030/// see [`Runner::gate`], which retries rather than records `Blocked` when
12031/// this is true.
12032fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12033    outcomes.iter().any(|o| o.resource_blocked)
12034}
12035
12036/// What [`Runner::gate_fix_round`] decided.
12037pub(crate) enum GateFix {
12038    /// The tree changed and `verify.e2e` is still green: run the gate again.
12039    Retry,
12040    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
12041    /// last failure stands and the run ends blocked.
12042    Stop,
12043    /// `verify.e2e` could not run after the fix (magi's own contention):
12044    /// decide nothing now, a later reentry retries.
12045    Defer,
12046}
12047
12048/// Is every red command in `outcomes` an ordinary failure the code could
12049/// explain: it ran, exited non-zero, and said something?
12050///
12051/// A timeout, a spawn failure and a killed process all leave `code` `None`;
12052/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
12053/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
12054/// what the command is stays the gate's business.
12055pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12056    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12057    red.peek().is_some()
12058        && red.all(|o| {
12059            !o.resource_blocked
12060                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12061                && !o.output_tail.trim().is_empty()
12062        })
12063}
12064
12065/// Describe one verify command's outcome for the event log, distinguishing a
12066/// build/link failure — the toolchain never produced a binary to run — from
12067/// an actual test failure, since only the latter is a verdict on the patch.
12068fn e2e_outcome_label(o: &CommandOutcome) -> String {
12069    if o.ok() {
12070        return "pass".to_owned();
12071    }
12072    let reason = if o.build_failed() {
12073        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12074    } else {
12075        format!("FAIL ({:?})", o.code)
12076    };
12077    format!(
12078        "{reason}\n{}",
12079        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12080    )
12081}
12082
12083/// Run `verify.e2e`, retrying once if the first attempt could not build or
12084/// link — a build/link failure is frequently a race against a shared
12085/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
12086/// `verify` event per command, tagged with `context` (normally `"round N"`)
12087/// so the two call sites that need this — the ordinary per-round leg in
12088/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
12089/// it will ever call a round green — read identically in the event log.
12090async fn run_e2e_with_retry(
12091    state: &mut RunState,
12092    shell: &[String],
12093    commands: &[String],
12094    worktree: &Path,
12095    timeout: Duration,
12096    context: &str,
12097) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12098    let (mut e2e, mut timed_out_pids) = run_commands(
12099        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12100    )
12101    .await;
12102    for o in &e2e {
12103        state.event(
12104            "verify",
12105            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12106        );
12107    }
12108    // A build/link failure is not a verdict on the patch — it is frequently a
12109    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
12110    // one retry before letting a red like that decide the round.
12111    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12112    if verify_retried {
12113        state.event(
12114            "verify",
12115            format!(
12116                "{context}: verify could not build/link, not a test result — retrying once \
12117                 before concluding"
12118            ),
12119        );
12120        let retried = run_commands(
12121            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12122        )
12123        .await;
12124        e2e = retried.0;
12125        // Both attempts' timeouts matter, not just the last one: the first
12126        // attempt's descendants may still be alive alongside the retry's.
12127        timed_out_pids.extend(retried.1);
12128        for o in &e2e {
12129            state.event(
12130                "verify",
12131                format!(
12132                    "{context}: retry `{}` -> {}",
12133                    o.command,
12134                    e2e_outcome_label(o)
12135                ),
12136            );
12137        }
12138    }
12139    (e2e, verify_retried, timed_out_pids)
12140}
12141
12142/// Run configured shell commands in `cwd`, in order. The second element is
12143/// the pid of every command that hit `timeout` and was still running when
12144/// this stopped waiting on it (best-effort: `None` when the platform did not
12145/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12146/// that releases a shared resource afterward needs to know.
12147///
12148/// Records `task` into [`RunState::active`] at every command boundary
12149/// (`RunState::task_command`) and clears it once the whole list has run
12150/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12151/// for minutes with no seat and no output of its own to show for it (see
12152/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12153/// apart from "not yet run" without this), and this is the only place that
12154/// knows which command is running right now and how many are left. Three
12155/// saves per command — start, not per second — matching the same "only at a
12156/// boundary" rule [`wave`] already follows for seats.
12157#[allow(clippy::too_many_arguments)]
12158async fn run_commands(
12159    state: &mut RunState,
12160    node: &str,
12161    task: &str,
12162    attempt: usize,
12163    shell: &[String],
12164    commands: &[String],
12165    cwd: &Path,
12166    timeout: Duration,
12167) -> (Vec<CommandOutcome>, Vec<u32>) {
12168    if commands.is_empty() {
12169        // Nothing to mark as running and nothing to clear — an empty list
12170        // means "not configured", and touching `active` (or the disk) over
12171        // that would be a write for every round of a repo with no
12172        // `verify.e2e` / `verify.gate` commands at all.
12173        return (Vec::new(), Vec::new());
12174    }
12175    let mut out = Vec::new();
12176    let mut timed_out_pids = Vec::new();
12177    let total = commands.len();
12178    for (idx, command) in commands.iter().enumerate() {
12179        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12180        if let Err(e) = state.save() {
12181            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12182        }
12183        let started = Instant::now();
12184        let mut cmd = tokio::process::Command::new(&shell[0]);
12185        cmd.quiet();
12186        // Verify the checkout as a standalone caller. Worker ownership and
12187        // pinned executable selection belong to this driver, not to tests or
12188        // tools it runs. Preserve home/seat context and Running's containment.
12189        for key in crate::worker::INVOCATION_ENV {
12190            cmd.env_remove(key);
12191        }
12192        cmd.args(&shell[1..])
12193            .arg(command)
12194            .current_dir(cwd)
12195            .stdin(std::process::Stdio::null())
12196            .stdout(std::process::Stdio::piped())
12197            .stderr(std::process::Stdio::piped())
12198            .kill_on_drop(true);
12199        let mut outputs = Vec::new();
12200        let protected = (|| -> Result<()> {
12201            crate::incremental::configure(&mut cmd, cwd)?;
12202            let mut paths = vec![cwd.to_owned()];
12203            if let Some(cache) = state.config.cache_dir() {
12204                paths.push(cache);
12205            }
12206            outputs.extend(crate::retention::OutputLease::admit(
12207                &crate::run::home(),
12208                &paths,
12209                state.config.disk.min_free_bytes,
12210            )?);
12211            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12212                outputs.push(guard);
12213            }
12214            if let Some(cache) = state.config.cache_dir() {
12215                outputs.push(crate::retention::OutputLease::cache(
12216                    &crate::run::home(),
12217                    &cache,
12218                    false,
12219                )?);
12220            }
12221            Ok(())
12222        })();
12223        if let Err(e) = protected {
12224            let detail = format!("execution admission unavailable: {e:#}");
12225            state.resource_block = Some(crate::run::ResourceBlock {
12226                cause: crate::run::ResourceBlockCause::Other,
12227                node: node.to_owned(),
12228                kind: if detail.contains("disk admission") {
12229                    "disk admission"
12230                } else {
12231                    "output ownership"
12232                }
12233                .to_owned(),
12234                detail: detail.clone(),
12235                at: Timestamp::now(),
12236            });
12237            out.push(CommandOutcome {
12238                command: command.clone(),
12239                code: None,
12240                output_tail: detail,
12241                duration_ms: 0,
12242                resource_blocked: true,
12243            });
12244            break;
12245        }
12246        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12247        let (code, body) = match spawned {
12248            Ok(mut running) => {
12249                let pid = running.child().id();
12250                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12251                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12252                let mut paths = vec![cwd.to_owned()];
12253                if let Some(cache) = state.config.cache_dir() {
12254                    paths.push(cache);
12255                }
12256                let mut pressure = None;
12257                let status = tokio::select! {
12258                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12259                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12260                        pressure = Some(reason); None
12261                    }
12262                };
12263                let (code, failure) = match status {
12264                    Some(Ok(Ok(status))) => (
12265                        status.code(),
12266                        status
12267                            .code()
12268                            .is_none()
12269                            .then(|| format!("command terminated: {status}")),
12270                    ),
12271                    Some(Ok(Err(error))) => {
12272                        (None, Some(format!("failed waiting for command: {error}")))
12273                    }
12274                    Some(Err(_)) => {
12275                        // Leased cleanup confirms process-tree death itself.
12276                        // Unleased commands still need the caller's PID check
12277                        // after kill-on-drop, before releasing cache ownership.
12278                        if !running.coordinated()
12279                            && let Some(pid) = pid
12280                        {
12281                            timed_out_pids.push(pid);
12282                        }
12283                        (
12284                            None,
12285                            Some(format!("timed out after {}s", timeout.as_secs())),
12286                        )
12287                    }
12288                    None => (None, None),
12289                };
12290                running.finish().await;
12291                if let Some(reason) = pressure {
12292                    state.resource_block = Some(crate::run::ResourceBlock {
12293                        cause: crate::run::ResourceBlockCause::Other,
12294                        node: node.to_owned(),
12295                        kind: "disk pressure".to_owned(),
12296                        detail: reason.clone(),
12297                        at: Timestamp::now(),
12298                    });
12299                    out.push(CommandOutcome {
12300                        command: command.clone(),
12301                        code: None,
12302                        output_tail: reason,
12303                        duration_ms: started.elapsed().as_millis() as u64,
12304                        resource_blocked: true,
12305                    });
12306                    break;
12307                }
12308                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12309                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12310                if let Some(failure) = failure {
12311                    body.push('\n');
12312                    body.push_str(&failure);
12313                }
12314                (code, body)
12315            }
12316            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12317        };
12318        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12319        if !body.is_empty() {
12320            // The full text is kept whatever the summary dropped. A fresh
12321            // name per command so a resume or retry never overwrites an
12322            // earlier run's output. A failed write changes no verdict.
12323            match crate::run::write_verification_artifact(state, task, &body) {
12324                Ok(name) if summary != body => {
12325                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12326                }
12327                Ok(_) => {}
12328                Err(e) => {
12329                    let warning = format!("could not save the full {task} output: {e:#}");
12330                    tracing::warn!("{warning}");
12331                    state.event(node, warning);
12332                    if summary != body {
12333                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12334                    }
12335                }
12336            }
12337        }
12338        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12339        out.push(CommandOutcome {
12340            command: command.clone(),
12341            code,
12342            output_tail: summary,
12343            duration_ms: started.elapsed().as_millis() as u64,
12344            resource_blocked,
12345        });
12346        if resource_blocked {
12347            state.resource_block = Some(crate::run::ResourceBlock {
12348                cause: crate::run::ResourceBlockCause::Other,
12349                node: node.to_owned(),
12350                kind: "incremental cache admission".into(),
12351                detail: body,
12352                at: Timestamp::now(),
12353            });
12354            break;
12355        }
12356    }
12357    state.task_finished(task);
12358    if let Err(e) = state.save() {
12359        tracing::warn!("could not persist the end of {task}: {e:#}");
12360    }
12361    (out, timed_out_pids)
12362}
12363
12364/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12365/// section (`report::run`) and in the `merge` event this node records — for
12366/// the operator to run by hand.
12367///
12368/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12369/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12370/// merge commits", or "require linear history") rejects the push a `--no-ff`
12371/// merge would produce, which is exactly the guidance this function replaces.
12372/// `message`'s first line becomes the squash commit's subject, matching the
12373/// note `report::run` prints alongside this command — see that function for
12374/// why an explicit subject is not optional there.
12375fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12376    let repo = repo.display();
12377    match style {
12378        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12379        MergeStyle::Squash => {
12380            // The subject sits inside double quotes, and a title an agent
12381            // wrote may carry the characters that break out of them.
12382            let subject = message
12383                .lines()
12384                .next()
12385                .unwrap_or(branch)
12386                .replace(['\\', '"', '$', '`'], "");
12387            format!(
12388                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12389            )
12390        }
12391        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12392    }
12393}
12394
12395/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12396/// under the hood, rejects a `title` over 256 characters and the whole
12397/// command fails — no PR at all, for a run whose body was otherwise fine
12398/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12399/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12400/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12401/// this bound. It is a margin, not a guarantee — a title packed
12402/// with multi-unit characters could still in principle land close to the
12403/// edge, but a real task title's occasional emoji or accented letter fits
12404/// comfortably inside it.
12405const PR_TITLE_MAX: usize = 240;
12406
12407/// A pull request title from the opening line of `text`, or `None` when that
12408/// line is not English (GitHub text is) or has no letters.
12409///
12410/// A line within `max` is kept as is. A longer one is cut at the end of its
12411/// first sentence when that falls inside `max`, else at a word boundary with a
12412/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12413/// make every merely-truncated title look non-English). The language check
12414/// runs on the kept text before any mark is added, so only what GitHub will
12415/// show is judged: an English opening followed by non-ASCII far past the cut
12416/// still passes.
12417fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12418    let line = queue::first_line(text)?;
12419    let chars: Vec<char> = line.chars().collect();
12420    let (kept, mark) = if chars.len() <= max {
12421        (line.to_owned(), "")
12422    } else if let Some(end) = sentence_end(&chars, max) {
12423        (chars[..end].iter().collect::<String>(), "")
12424    } else {
12425        let room = max.saturating_sub(3);
12426        // Cut at the last space inside the room; when the char just past the
12427        // room is a space the room already ends on a word.
12428        let cut = if chars[room].is_whitespace() {
12429            room
12430        } else {
12431            chars[..room]
12432                .iter()
12433                .rposition(|c| c.is_whitespace())
12434                .unwrap_or(room)
12435        };
12436        let head: String = chars[..cut].iter().collect();
12437        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12438        (head.to_owned(), "...")
12439    };
12440    if kept.is_empty() {
12441        return None;
12442    }
12443    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12444        return None;
12445    }
12446    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12447        return None;
12448    }
12449    Some(format!("{kept}{mark}"))
12450}
12451
12452/// The char length of the first sentence of `chars` when it ends within `max`
12453/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12454/// abbreviations so `e.g. foo` does not end a title early.
12455fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12456    const MIN: usize = 20;
12457    for i in MIN..max.min(chars.len()) {
12458        if !matches!(chars[i], '.' | '!' | '?') {
12459            continue;
12460        }
12461        let Some(&next) = chars.get(i + 1) else {
12462            continue;
12463        };
12464        if !next.is_whitespace() {
12465            continue;
12466        }
12467        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12468        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12469            continue;
12470        }
12471        let word: String = chars[..i]
12472            .iter()
12473            .rev()
12474            .take_while(|c| !c.is_whitespace())
12475            .collect::<Vec<_>>()
12476            .into_iter()
12477            .rev()
12478            .collect();
12479        let word = word.to_ascii_lowercase();
12480        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12481            continue;
12482        }
12483        let end = chars[..i]
12484            .iter()
12485            .rposition(|c| !c.is_whitespace())
12486            .map_or(i, |p| p + 1);
12487        return Some(end);
12488    }
12489    None
12490}
12491
12492/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12493/// change: a title and a body describing what was *implemented*, not the task
12494/// that asked for it. A task reads as a request; a reader of the merged
12495/// history wants the change.
12496struct PrMessage {
12497    title: String,
12498    body: String,
12499}
12500
12501impl PrMessage {
12502    /// Title, blank line, body. The first line is the squash/merge commit
12503    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12504    /// has to stay one sensible line.
12505    fn commit_message(&self) -> String {
12506        format!("{}\n\n{}", self.title, self.body)
12507    }
12508}
12509
12510/// The text after a leading `TITLE:` (any case) on `line`.
12511fn title_marker(line: &str) -> Option<&str> {
12512    let line = line.trim();
12513    let head = line.get(..6)?;
12514    head.eq_ignore_ascii_case("title:")
12515        .then(|| line[6..].trim())
12516}
12517
12518/// The implementer's own one-line title: the `TITLE:` line the implement
12519/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12520/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12521/// source, and a title that says as much is refused here too.
12522fn summary_title(summary: &str) -> Option<String> {
12523    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12524    let raw = title_marker(first)?;
12525    if raw.is_empty() {
12526        return None;
12527    }
12528    let title = queue::title_from(raw, PR_TITLE_MAX);
12529    let lower = title.to_ascii_lowercase();
12530    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12531        return None;
12532    }
12533    Some(title)
12534}
12535
12536/// How `open_review`'s instruction begins; see [`landing_title`].
12537const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12538
12539/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12540pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12541
12542/// Does this run review work that already existed, rather than implement a
12543/// task? Runs recorded before `reviewed_commits` existed carry only the
12544/// candidate marker.
12545pub(crate) fn is_review_run(state: &RunState) -> bool {
12546    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12547}
12548
12549/// The title of a review-only run: the subject of the oldest commit under
12550/// review. Later commits are usually fixups, and `instruction` is the review
12551/// prompt, which says nothing about the change. GitHub text is English, so a
12552/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12553fn review_title(state: &RunState) -> Option<String> {
12554    english_subject(
12555        state.reviewed_commits.as_ref()?.first()?,
12556        state.config.graph.github_english(),
12557    )
12558}
12559
12560/// `raw` as a pull request title, or `None` when it is blank, not English
12561/// (GitHub text is), or one of magi's own candidate commit subjects.
12562fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12563    let raw = raw.trim();
12564    if raw.is_empty() {
12565        return None;
12566    }
12567    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12568        return None;
12569    }
12570    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12571        return None;
12572    }
12573    let title = queue::title_from(raw, PR_TITLE_MAX);
12574    let lower = title.to_ascii_lowercase();
12575    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12576        return None;
12577    }
12578    Some(title)
12579}
12580
12581/// What a review-only run's branch says about itself, read at the moment the
12582/// pull request is opened.
12583#[derive(Debug, Clone, PartialEq, Eq)]
12584struct BranchFacts {
12585    /// `(subject, body)` of each commit, oldest first.
12586    commits: Vec<(String, String)>,
12587    /// Trimmed `git diff --stat`.
12588    stat: String,
12589}
12590
12591/// Longest diff stat shown: this many file lines plus the summary line.
12592const STAT_FILE_LINES: usize = 25;
12593/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12594const COMMITS_MAX_CHARS: usize = 20_000;
12595
12596/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12597/// say or finds nothing, so the caller falls back to what the run recorded.
12598async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12599    let commits = git::commit_log(repo, base, branch).await.ok()?;
12600    if commits.is_empty() {
12601        return None;
12602    }
12603    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12604    let lines: Vec<&str> = stat.lines().collect();
12605    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12606        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12607        let more = format!(" ... {omitted} more file(s)");
12608        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12609        kept.push(&more);
12610        kept.push(lines[lines.len() - 1]);
12611        kept.join("\n")
12612    } else {
12613        lines.join("\n")
12614    };
12615    Some(BranchFacts { commits, stat })
12616}
12617
12618/// Defang what would break the surrounding markdown: a closing `</details>`
12619/// and a code fence.
12620fn markdown_safe(text: &str) -> String {
12621    text.replace("</details>", "&lt;/details&gt;")
12622        .replace("\x60\x60\x60", "~~~")
12623}
12624
12625fn neutral_title(state: &RunState, winner: char) -> String {
12626    format!(
12627        "chore: land candidate {} of run {}",
12628        winner.to_ascii_uppercase(),
12629        state.id
12630    )
12631}
12632
12633/// Reuse the graph's bounded fixer without starting another graph or driver.
12634/// Copy the working state back even on errors so spent rounds remain visible.
12635pub(crate) async fn repair_landing_gate(
12636    state: &mut RunState,
12637    winner: &Candidate,
12638    outcomes: &[CommandOutcome],
12639) -> Result<GateFix> {
12640    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12641        return Ok(GateFix::Stop);
12642    }
12643    let mut runner = Runner {
12644        roles: state.config.resolve_roles()?,
12645        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12646        state: state.clone(),
12647        pause: Pause::new(),
12648        interrupt: Pause::new(),
12649    };
12650    let result = runner.gate_fix_round(winner, outcomes).await;
12651    *state = runner.state;
12652    result
12653}
12654
12655/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12656/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12657pub(crate) async fn verify_current_head(
12658    state: &mut RunState,
12659    winner: &Candidate,
12660    wait: &crate::repo_lock::WaitPolicy,
12661) -> Result<bool> {
12662    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12663    // Preserve the previous record before replacing it, including on crash recovery.
12664    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12665        state
12666            .verification_history
12667            .push(crate::run::VerificationEvidence {
12668                superseded_by: Some(head.clone()),
12669                archived_at: Some(Timestamp::now()),
12670                fixes_spent: Some(state.gate_fixes.len()),
12671                gate_head: None,
12672                gate_ran: false,
12673                gate: Vec::new(),
12674                e2e: state.head_e2e.take(),
12675            });
12676    }
12677    if let Some(e) = &state.head_e2e
12678        && e.head == head
12679    {
12680        if e.passed {
12681            return Ok(true);
12682        }
12683        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12684            state.status = RunStatus::Blocked;
12685            state.save()?;
12686            return Ok(false);
12687        }
12688    } else if state.reviews.last().is_some_and(|r| {
12689        r.verified_head.as_deref() == Some(head.as_str())
12690            && (r.e2e_status() == E2eStatus::Passed
12691                || (r.e2e_status() == E2eStatus::NotConfigured
12692                    && state.config.verify.e2e.is_empty()))
12693    }) || state
12694        .gate_repair_e2e
12695        .as_ref()
12696        .is_some_and(|e| e.head == head && e.passed)
12697    {
12698        return Ok(true);
12699    }
12700    if let Some(e) = &state.gate_repair_e2e
12701        && e.head == head
12702        && !e.passed
12703        && !e.outcomes.is_empty()
12704        && !verify_inconclusive(&e.outcomes)
12705    {
12706        state.status = RunStatus::Blocked;
12707        state.save()?;
12708        return Ok(false);
12709    }
12710    state.head_e2e = Some(crate::run::GateRepairE2e {
12711        head: head.clone(),
12712        passed: false,
12713        outcomes: Vec::new(),
12714    });
12715    state.save()?;
12716    let commands = state.config.verify.e2e.clone();
12717    let shell = state.config.shell();
12718    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12719    let cache_dir = state.config.cache_dir();
12720    let outcomes = if commands.is_empty() {
12721        Vec::new()
12722    } else {
12723        let (outcomes, _) = with_cache_lease(
12724            state,
12725            cache_dir.as_deref(),
12726            "e2e",
12727            "e2e",
12728            &winner.worktree,
12729            &head,
12730            timeout,
12731            "current head e2e",
12732            wait,
12733            |state, budget| async move {
12734                run_e2e_with_retry(
12735                    state,
12736                    &shell,
12737                    &commands,
12738                    &winner.worktree,
12739                    budget,
12740                    "current head e2e",
12741                )
12742                .await
12743            },
12744        )
12745        .await?;
12746        outcomes
12747    };
12748    let unresolved = verify_inconclusive(&outcomes);
12749    let passed = !unresolved
12750        && outcomes.iter().all(CommandOutcome::ok)
12751        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12752    if !outcomes.is_empty() && !unresolved {
12753        state.record_base_progress(&head);
12754    }
12755    state.head_e2e = Some(crate::run::GateRepairE2e {
12756        head: head.clone(),
12757        passed,
12758        outcomes,
12759    });
12760    if !passed && !unresolved {
12761        state.status = RunStatus::Blocked;
12762        state.event(
12763            "e2e",
12764            format!("verify.e2e failed on head {head}; nothing published"),
12765        );
12766    }
12767    state.save()?;
12768    Ok(passed)
12769}
12770
12771/// Recheck a repair's e2e requirement across failures, resource deferrals and
12772/// process interruptions. A successful result only applies to its exact SHA.
12773pub(crate) async fn check_gate_repair_e2e(
12774    state: &mut RunState,
12775    winner: &Candidate,
12776    wait: &crate::repo_lock::WaitPolicy,
12777) -> Result<GateFix> {
12778    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12779        return Ok(GateFix::Retry);
12780    }
12781    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12782    if state
12783        .gate_repair_e2e
12784        .as_ref()
12785        .is_some_and(|r| r.head == head && r.passed)
12786    {
12787        return Ok(GateFix::Retry);
12788    }
12789    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12790        head: head.clone(),
12791        passed: false,
12792        outcomes: Vec::new(),
12793    });
12794    state.save()?;
12795    let commands = state.config.verify.e2e.clone();
12796    let outcomes = if commands.is_empty() {
12797        Vec::new()
12798    } else {
12799        let shell = state.config.shell();
12800        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12801        let cache_dir = state.config.cache_dir();
12802        let (outcomes, _) = with_cache_lease(
12803            state,
12804            cache_dir.as_deref(),
12805            "e2e",
12806            "e2e",
12807            &winner.worktree,
12808            &head,
12809            timeout,
12810            "gate repair e2e",
12811            wait,
12812            |state, budget| async move {
12813                run_e2e_with_retry(
12814                    state,
12815                    &shell,
12816                    &commands,
12817                    &winner.worktree,
12818                    budget,
12819                    "gate repair e2e",
12820                )
12821                .await
12822            },
12823        )
12824        .await?;
12825        outcomes
12826    };
12827    let deferred = verify_inconclusive(&outcomes);
12828    let passed = outcomes.iter().all(CommandOutcome::ok)
12829        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12830    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12831        head,
12832        passed,
12833        outcomes,
12834    });
12835    state.save()?;
12836    if deferred {
12837        Ok(GateFix::Defer)
12838    } else if !passed {
12839        state.event(
12840            "gate",
12841            "verify.e2e failed after gate repair; nothing published",
12842        );
12843        Ok(GateFix::Stop)
12844    } else {
12845        Ok(GateFix::Retry)
12846    }
12847}
12848
12849/// Run the same configured gate for a landing head and a fresh graph.
12850pub(crate) async fn run_landing_gate(
12851    state: &mut RunState,
12852    winner: &Candidate,
12853    wait: &crate::repo_lock::WaitPolicy,
12854) -> Result<Vec<CommandOutcome>> {
12855    let shell = state.config.shell();
12856    let gate_commands = state.config.verify.gate.clone();
12857    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12858    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12859        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12860        Vec::new()
12861    } else {
12862        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12863        let cache_dir = state.config.cache_dir();
12864        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12865        let (outcomes, _) = with_cache_lease(
12866            state,
12867            cache_dir.as_deref(),
12868            "gate",
12869            "gate",
12870            &winner.worktree,
12871            &head,
12872            timeout,
12873            "final gate",
12874            wait,
12875            |state, budget| {
12876                let shell = shell.clone();
12877                let gate_commands = gate_commands.clone();
12878                let winner = winner.clone();
12879                async move {
12880                    Runner::run_pre_gate(state, &winner).await;
12881                    // Bind command evidence after mechanical fixes, before commands start.
12882                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12883                    let (outcomes, timed_out_pids) = run_commands(
12884                        state,
12885                        "gate",
12886                        "gate",
12887                        0,
12888                        &shell,
12889                        &gate_commands,
12890                        &winner.worktree,
12891                        budget,
12892                    )
12893                    .await;
12894                    (outcomes, false, timed_out_pids)
12895                }
12896            },
12897        )
12898        .await?;
12899        outcomes
12900    };
12901    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12902    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12903        let mut outcomes = outcomes;
12904        outcomes.push(CommandOutcome {
12905            command: "gate head stability".into(), code: None,
12906            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12907            duration_ms: 0, resource_blocked: false,
12908        });
12909        return Ok(outcomes);
12910    }
12911    if outcomes.is_empty() {
12912        // Nothing configured to check — distinct from every other
12913        // silence in this run's event log, since an empty `gate` alone
12914        // no longer says whether the gate ran at all (see
12915        // `RunState::gate_ran`'s own doc).
12916        state.event("gate", "no gate commands configured; not tested");
12917    }
12918    for o in &outcomes {
12919        state.event(
12920            "gate",
12921            format!(
12922                "`{}` -> {}",
12923                o.command,
12924                if o.ok() {
12925                    "pass".to_owned()
12926                } else {
12927                    format!(
12928                        "FAIL ({:?})\n{}",
12929                        o.code,
12930                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12931                    )
12932                }
12933            ),
12934        );
12935    }
12936    Ok(outcomes)
12937}
12938
12939/// The pull request title to hand to `land::merge_subject`. A review-only run
12940/// opened by an earlier build titled its pull request with the review prompt;
12941/// that title is dropped (empty, so the fallback applies) rather than landed.
12942/// Any other title, including an operator's rename, passes through untouched,
12943/// and so does every title of a run that implements a task.
12944pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12945    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12946        ""
12947    } else {
12948        pr_title
12949    }
12950}
12951
12952/// What the squash subject falls back to when the pull request title is empty
12953/// or candidate-shaped: for a review-only run the derived title, never the
12954/// review prompt held in `instruction`.
12955pub fn landing_subject_source(state: &RunState) -> String {
12956    if is_review_run(state) {
12957        let winner = state.candidates.first().map_or('A', |c| c.label);
12958        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12959    }
12960    state.instruction.clone()
12961}
12962
12963/// `summary` without its `TITLE:` line, which the pull request title already
12964/// carries.
12965fn summary_without_title(summary: &str) -> String {
12966    let mut lines = summary.trim().lines().peekable();
12967    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12968        lines.next();
12969    }
12970    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12971}
12972
12973/// The pull request title and body for the winning candidate.
12974///
12975/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12976/// the task's own opening line via [`queue::title_from`] when there is none.
12977/// `state.instruction` can open with blank lines (`task_text` only rejects a
12978/// body that is blank *entirely*), which `title_from` skips.
12979///
12980/// Publishing uses an independent description generated from the final diff
12981/// and structured evidence. Internal summaries and reports remain local.
12982/// Non-publishing previews may display the original task and candidate summary.
12983/// The footer retains the run and candidate provenance.
12984#[cfg(test)]
12985fn pr_message(state: &RunState, winner: char) -> PrMessage {
12986    pr_message_with(state, winner, None)
12987}
12988
12989/// [`pr_message`] with what the branch of a review-only run says about itself.
12990/// `facts` is ignored for a run that implements a task.
12991#[cfg(test)]
12992fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12993    let raw = pr_message_raw(
12994        state,
12995        winner,
12996        facts,
12997        crate::pr_outcome::FinalHead::Unreported,
12998        ChangeNote::NotAsked,
12999    );
13000    let id = crate::scrub::Identity::current();
13001    PrMessage {
13002        title: crate::scrub::scrub(&raw.title, &id),
13003        body: crate::scrub::scrub(&raw.body, &id),
13004    }
13005}
13006
13007/// What the public description may say about the change itself.
13008#[derive(Debug, Clone, Copy)]
13009enum ChangeNote<'a> {
13010    /// A local preview or non-publishing merge mode: no public generation needed.
13011    NotAsked,
13012    /// A summary written from the final diff.
13013    Summary(&'a crate::run::ChangeSummary),
13014    /// A summary was asked for and could not be produced; the text says why.
13015    Unavailable(&'a str),
13016}
13017
13018fn pr_message_raw(
13019    state: &RunState,
13020    winner: char,
13021    facts: Option<&BranchFacts>,
13022    head: crate::pr_outcome::FinalHead<'_>,
13023    change: ChangeNote<'_>,
13024) -> PrMessage {
13025    if !matches!(change, ChangeNote::NotAsked) {
13026        let (title, mut body) = match change {
13027            ChangeNote::Summary(c) => (
13028                c.title.clone(),
13029                format!("## Summary\n\n{}\n\n", c.body.trim()),
13030            ),
13031            ChangeNote::Unavailable(why) => (
13032                neutral_title(state, winner),
13033                format!(
13034                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13035                ),
13036            ),
13037            ChangeNote::NotAsked => unreachable!(),
13038        };
13039        body.push_str(&public_pr_evidence(state, head));
13040        body.push_str(&format!(
13041            "\n---\nmagi:run/{} magi:candidate-{}\n",
13042            state.id,
13043            winner.to_ascii_lowercase()
13044        ));
13045        return PrMessage { title, body };
13046    }
13047    pr_preview(state, winner, facts, head)
13048}
13049
13050/// Append public facts from the run records after every prose generation,
13051/// including posting-gate rewrites. The agent cannot drop this provenance.
13052fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13053    let mut body = crate::pr_outcome::review_rounds(state);
13054    body.push_str(&crate::pr_outcome::final_verification(state, head));
13055    let open = state.last_round_findings();
13056    if !open.is_empty() {
13057        body.push_str("## Open review findings\n\n");
13058        for f in open {
13059            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13060        }
13061        body.push('\n');
13062    }
13063    if let Some(c) = state.change_summary.as_ref()
13064        && head == crate::pr_outcome::FinalHead::At(&c.head)
13065    {
13066        body.push_str(&format!(
13067            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13068            c.base, c.head
13069        ));
13070    }
13071    body
13072}
13073
13074/// Local-only preview of the operator's task and candidate summary.
13075fn pr_preview(
13076    state: &RunState,
13077    winner: char,
13078    facts: Option<&BranchFacts>,
13079    head: crate::pr_outcome::FinalHead<'_>,
13080) -> PrMessage {
13081    let summary = state
13082        .candidates
13083        .iter()
13084        .find(|c| c.label == winner)
13085        .map(|c| c.summary.as_str())
13086        .unwrap_or_default();
13087    // The fallback is the operator's own words and may not be English; GitHub
13088    // text always is, so a non-English task gets a neutral title instead.
13089    let review = is_review_run(state);
13090    let title = if review {
13091        facts
13092            .and_then(|f| {
13093                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13094            })
13095            .or_else(|| review_title(state))
13096            .or_else(|| {
13097                state
13098                    .candidates
13099                    .iter()
13100                    .find(|c| c.label == winner)
13101                    .filter(|c| !c.branch.starts_with("magi/"))
13102                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13103            })
13104            .unwrap_or_else(|| neutral_title(state, winner))
13105    } else {
13106        summary_title(summary).unwrap_or_else(|| {
13107            english_title(
13108                &state.instruction,
13109                PR_TITLE_MAX,
13110                state.config.graph.github_english(),
13111            )
13112            .unwrap_or_else(|| neutral_title(state, winner))
13113        })
13114    };
13115
13116    let mut body = String::new();
13117    let what = summary_without_title(summary);
13118    if !what.is_empty() {
13119        body.push_str("## Summary\n\n");
13120        body.push_str(&what);
13121        body.push_str("\n\n");
13122    }
13123
13124    // Review history uses structured facts across all rounds, never reports.
13125    body.push_str(&crate::pr_outcome::review_rounds(state));
13126    body.push_str(&crate::pr_outcome::final_verification(state, head));
13127    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13128
13129    let open = state.open_findings();
13130    if !open.is_empty() {
13131        body.push_str("## Open review findings\n\n");
13132        for f in &open {
13133            body.push_str(&format!(
13134                "- `{}` [{:?}] {}\n",
13135                f.id, f.severity, "Context retained in local review records."
13136            ));
13137        }
13138        body.push('\n');
13139    }
13140
13141    if let Some(fix) = fix
13142        && !fix.rejected.is_empty()
13143    {
13144        body.push_str("## Declined by the fixer\n\n");
13145        for r in &fix.rejected {
13146            body.push_str(&format!(
13147                "- `{}`: {}\n",
13148                r.id, "Reason retained in local review records."
13149            ));
13150        }
13151        body.push('\n');
13152    }
13153
13154    if review {
13155        // The review prompt is not the task; list what the branch carries.
13156        body.push_str("## Commits under review\n\n");
13157        if let Some(facts) = facts {
13158            let mut left = COMMITS_MAX_CHARS;
13159            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13160                let mut entry = format!("- {}\n", markdown_safe(subject));
13161                for l in markdown_safe(text).lines() {
13162                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13163                }
13164                if left == 0 {
13165                    body.push_str(&format!(
13166                        "- ... {} more commit(s)\n",
13167                        facts.commits.len() - i
13168                    ));
13169                    break;
13170                }
13171                if entry.len() > left {
13172                    // Even the first commit is cut: one huge body must not
13173                    // push the whole description past GitHub's limit.
13174                    let mut end = left;
13175                    while !entry.is_char_boundary(end) {
13176                        end -= 1;
13177                    }
13178                    entry.truncate(end);
13179                    entry.push_str("\n  ... (truncated)\n");
13180                    left = 0;
13181                } else {
13182                    left -= entry.len();
13183                }
13184                body.push_str(&entry);
13185            }
13186            if !facts.stat.trim().is_empty() {
13187                body.push_str(&format!(
13188                    "\n## Diff stat\n\n```\n{}\n```\n",
13189                    markdown_safe(facts.stat.trim())
13190                ));
13191            }
13192        } else {
13193            match &state.reviewed_commits {
13194                Some(subjects) => {
13195                    for s in subjects {
13196                        body.push_str(&format!("- {}\n", s.trim()));
13197                    }
13198                }
13199                None => {
13200                    // An older run kept only the prompt, with the commit list
13201                    // after its first paragraph.
13202                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13203                    body.push_str(rest.trim());
13204                    body.push('\n');
13205                }
13206            }
13207        }
13208    } else {
13209        let task = state.instruction.trim();
13210        let task = if task.is_empty() {
13211            "(empty task)"
13212        } else {
13213            task
13214        };
13215        body.push_str(&format!(
13216            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13217            task.replace("</details>", "&lt;/details&gt;")
13218        ));
13219    }
13220
13221    body.push_str(&format!(
13222        "\n---\nmagi:run/{} magi:candidate-{}\n",
13223        state.id,
13224        winner.to_ascii_lowercase()
13225    ));
13226
13227    PrMessage { title, body }
13228}
13229
13230/// Source imports have already been prepared from structured inputs. Legacy
13231/// inferred references must never instruct a seat to reconstruct those imports.
13232fn seeded_instruction(state: &RunState) -> String {
13233    let mut inputs = Vec::new();
13234    if let Some(start) = &state.sources.start_from {
13235        inputs.push(format!(
13236            "Start from `{}` at `{}` (including ancestry).",
13237            start.token, start.sha
13238        ));
13239    }
13240    for pick in &state.sources.pick_commits {
13241        inputs.push(format!(
13242            "Pick `{}` at `{}` (single commit).",
13243            pick.token, pick.sha
13244        ));
13245    }
13246    if inputs.is_empty() {
13247        state.instruction.clone()
13248    } else {
13249        format!(
13250            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13251            state.instruction,
13252            inputs.join("\n")
13253        )
13254    }
13255}
13256
13257/// Does the winner have no commits ahead of the base it would land on?
13258/// Any failure to find out reads as "not empty": the merge then behaves as it
13259/// always did rather than refusing on a guess.
13260async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13261    let base = &state.base_branch;
13262    let mut against = base.clone();
13263    if mode == MergeMode::Pr {
13264        // A pull request lands on the remote's base, never the local branch
13265        // of the same name: if that cannot be read, "not empty" is the safe
13266        // answer.
13267        let remote = &state.config.merge.remote;
13268        let tracking = format!("{remote}/{base}");
13269        let fetched = git::fetch(repo, remote, base).await;
13270        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13271            against = tracking;
13272        } else {
13273            return false;
13274        }
13275    }
13276    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13277}
13278
13279/// Why nothing was opened for an empty winner, with what the task's own
13280/// references resolved to.
13281fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13282    let mut detail = format!(
13283        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13284         nothing to open a pull request for"
13285    );
13286    detail.push_str(
13287        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13288    );
13289    detail
13290}
13291
13292/// What the `Pr` merge does once it knows whether the branch already has an
13293/// open pull request.
13294#[derive(Debug, PartialEq, Eq)]
13295enum PrPlan {
13296    Create,
13297    Adopt { url: String, title: String },
13298    Stop(String),
13299}
13300
13301/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13302/// a failed lookup -> stop with the real reason. Never guesses.
13303fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13304    match found {
13305        Ok(land::OpenPr::None) => PrPlan::Create,
13306        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13307        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13308            "several open pull requests exist for this branch, not picking one: {}",
13309            urls.join(" ")
13310        )),
13311        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13312    }
13313}
13314
13315/// `gh pr create`, returning the PR url.
13316pub(crate) async fn gh_pr_create(
13317    cwd: &Path,
13318    base: &str,
13319    head: &str,
13320    title: &str,
13321    body: &str,
13322) -> Result<String> {
13323    let out = tokio::process::Command::new("gh")
13324        .args([
13325            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13326        ])
13327        .current_dir(cwd)
13328        .quiet()
13329        .stdin(std::process::Stdio::null())
13330        .output()
13331        .await
13332        .context("spawn gh")?;
13333    if out.status.success() {
13334        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13335    } else {
13336        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13337    }
13338}
13339
13340/// Never force deletion of uncommitted work, including unknown legacy trees.
13341async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13342    match std::fs::symlink_metadata(path) {
13343        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13344            // A previous cleanup may have removed the tree before a branch
13345            // deletion or state save failed. Unregister it if still present.
13346            git::worktree_remove_clean(repo, path).await?;
13347            return Ok(());
13348        }
13349        Err(e) => return Err(e.into()),
13350        Ok(_) => {}
13351    }
13352    if !git::is_clean(path).await? {
13353        bail!(
13354            "worktree has uncommitted work; refusing removal: {}",
13355            path.display()
13356        );
13357    }
13358    if !git::worktree_remove_clean(repo, path).await? {
13359        bail!("worktree removal refused: {}", path.display());
13360    }
13361    Ok(())
13362}
13363
13364/// Read seats return replies, not working-tree changes. Only registered,
13365/// detached seats directly inside this run's root may discard those changes.
13366async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13367    if !crate::retention::read_seat(path) || !path.exists() {
13368        return remove_preserved_worktree(&state.repo, path).await;
13369    }
13370    let owned = std::fs::symlink_metadata(path)?.is_dir()
13371        && path.parent().map(Path::canonicalize).transpose()?
13372            == Some(state.worktree_root().canonicalize()?)
13373        && crate::retention::worktrees(state)
13374            .await?
13375            .contains(&path.canonicalize()?);
13376    let common = crate::repo_lock::common_dir(&state.repo).await?;
13377    if !owned
13378        || common.is_none()
13379        || crate::repo_lock::common_dir(path).await? != common
13380        || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13381            .await?
13382            .ok()
13383        || state.candidates.iter().any(|c| c.worktree == path)
13384    {
13385        return remove_preserved_worktree(&state.repo, path).await;
13386    }
13387    let changes = git::status_porcelain(path).await?;
13388    if !git::worktree_remove(&state.repo, path).await? {
13389        bail!(
13390            "read-only seat worktree removal refused: {}",
13391            path.display()
13392        );
13393    }
13394    if !changes.is_empty() {
13395        let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13396        let paths: String = paths.chars().take(1024).collect();
13397        state.event(
13398            "fold",
13399            format!(
13400                "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13401                path.file_name().unwrap_or_default().to_string_lossy(),
13402                changes.lines().count()
13403            ),
13404        );
13405    }
13406    Ok(())
13407}
13408
13409/// Tear a run's worktrees and branches down.
13410///
13411/// `home` is where the updated `run.json` is saved (via
13412/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13413/// a housekeeping pass already has its own honest `home` handed to it, and
13414/// falling through to the global here would write back through whichever
13415/// directory some other process or test pinned into that `OnceLock` first,
13416/// not the one the caller actually resolved its `runs` and `state` from.
13417pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13418    fold_run_with_policy(state, drop_winner, home, false).await
13419}
13420
13421/// Explicit CLI-only retention override; automatic callers use [`fold_run`].
13422pub async fn fold_run_overriding_recovery_retention(
13423    state: &mut RunState,
13424    drop_winner: bool,
13425    home: &Path,
13426) -> Result<Vec<String>> {
13427    fold_run_with_policy(state, drop_winner, home, true).await
13428}
13429
13430async fn fold_run_with_policy(
13431    state: &mut RunState,
13432    drop_winner: bool,
13433    home: &Path,
13434    override_recovery_retention: bool,
13435) -> Result<Vec<String>> {
13436    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state)
13437        && !override_recovery_retention
13438    {
13439        bail!("refusing fold: {reason}");
13440    }
13441    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13442        bail!("candidate preservation incomplete; refusing cleanup");
13443    }
13444    // Serialize ordinary folding with the same starts/resumes as target-only
13445    // eviction. Otherwise a second janitor could remove the whole worktree
13446    // around a live output borrower even though target eviction was fenced.
13447    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13448    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13449    // An override must use current durable ownership, not a pre-lock snapshot.
13450    if override_recovery_retention {
13451        *state = RunState::load_under(&state.id, home)?;
13452        anyhow::ensure!(
13453            state.status.done() && !state.status.resumable(),
13454            "recovery retention override requires a finished, non-resumable run"
13455        );
13456        let claimed = crate::daemon::is_working_on_other(home, &state.id, Timestamp::now())?;
13457        anyhow::ensure!(
13458            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
13459            "run is live or its ownership is unknown"
13460        );
13461        anyhow::ensure!(
13462            state.candidate_preservation.is_empty() && state.interrupted_implementations.is_empty(),
13463            "candidate preservation incomplete; refusing cleanup"
13464        );
13465    }
13466    let retention_reason = crate::pr_recovery::source_retention_reason(home, state);
13467    if let Some(reason) = &retention_reason
13468        && !override_recovery_retention
13469    {
13470        bail!("refusing fold: {reason}");
13471    }
13472    let _driver = crate::driver::maintenance(home, &state.id)?;
13473    let mut output_guards = Vec::new();
13474    for entry in std::fs::read_dir(state.worktree_root())
13475        .into_iter()
13476        .flatten()
13477        .flatten()
13478    {
13479        if entry.path().join(".git").is_file() {
13480            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13481        }
13482    }
13483    let repo = state.repo.clone();
13484    // CLI, Web and automatic cleanup all enter here, independently of
13485    // Runner::fold_losers. Protect deletion and shared config release too.
13486    let wait = repository_wait(state, None);
13487    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13488        Ok(lease) => lease,
13489        Err(e) => {
13490            state.save_under(home)?;
13491            return Err(e);
13492        }
13493    };
13494    // Confirmed merges dispose of their owned sources after durable evidence
13495    // capture. Preflight every source before removing anything.
13496    let _source_outputs = if retention_reason.is_some()
13497        || (state.status == RunStatus::Merged && state.merge.as_ref().is_some_and(|merge| merge.ok))
13498    {
13499        verify_merged_fold_sources(state, drop_winner, home).await?
13500    } else {
13501        Vec::new()
13502    };
13503    if let Some(reason) = retention_reason {
13504        state.event(
13505            "fold",
13506            format!(
13507                "operator CLI --override-recovery-retention authorized deletion for run {}: bypassed {reason}",
13508                state.id
13509            ),
13510        );
13511        state.save_under(home)?;
13512    }
13513    let root = state.worktree_root();
13514    let winner = state.tally.as_ref().map(|t| t.winner);
13515    let mut removed = Vec::new();
13516
13517    for i in 0..state.candidates.len() {
13518        let c = state.candidates[i].clone();
13519        let is_winner = Some(c.label) == winner;
13520        if is_winner && !drop_winner {
13521            continue;
13522        }
13523        let existed = c.worktree.exists();
13524        // A vanished directory can still have a Git registration that keeps
13525        // its branch checked out. Remove that registration before the branch.
13526        remove_preserved_worktree(&repo, &c.worktree).await?;
13527        if existed {
13528            removed.push(c.worktree.to_string_lossy().into_owned());
13529        }
13530        // A branch handed to a later run (and its pull request) is not this
13531        // run's to delete.
13532        let handed_over = state.released_branches.contains(&c.branch);
13533        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13534            if !git::branch_delete(&repo, &c.branch).await? {
13535                bail!("candidate branch removal refused");
13536            }
13537            removed.push(c.branch.clone());
13538        }
13539        state.candidates[i].folded = true;
13540    }
13541
13542    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13543        let path = name.path();
13544        let keep = !drop_winner
13545            && winner.is_some_and(|w| {
13546                path.file_name()
13547                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13548            });
13549        if keep {
13550            continue;
13551        }
13552        remove_read_seat_worktree(state, &path).await?;
13553        removed.push(path.to_string_lossy().into_owned());
13554    }
13555
13556    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13557    // judge worktrees, so once the loop above has cleared all of them out,
13558    // the parent is a bare directory nobody else was ever going to remove -
13559    // git only ever managed what was inside it. Left alone, one of these
13560    // accumulates per fully-folded run; the operator's own machine had 74.
13561    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13562    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13563    // so does anything a run never claimed that happens to share the bay.
13564    remove_if_empty(&root);
13565
13566    if state.enabled_worktree_config && drop_winner {
13567        // A release, not a raw disable: some sibling run in this repository
13568        // may still hold its own reference (see `git::acquire_worktree_config`),
13569        // and only the last release actually turns the setting back off.
13570        git::release_worktree_config(&repo).await.ok();
13571        state.enabled_worktree_config = false;
13572    }
13573    state.save_under(home)?;
13574    Ok(removed)
13575}
13576
13577async fn verify_merged_fold_sources(
13578    state: &RunState,
13579    drop_winner: bool,
13580    home: &Path,
13581) -> Result<Vec<std::fs::File>> {
13582    let winner = state.tally.as_ref().map(|t| t.winner);
13583    let mut heads = BTreeSet::new();
13584    for candidate in &state.candidates {
13585        if Some(candidate.label) == winner && !drop_winner {
13586            continue;
13587        }
13588        anyhow::ensure!(
13589            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13590            "candidate recovery source is outside the inventoried run root: {}",
13591            candidate.worktree.display()
13592        );
13593        match std::fs::symlink_metadata(&candidate.worktree) {
13594            Ok(_) => {}
13595            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13596            Err(error) => return Err(error.into()),
13597        }
13598        if !state.released_branches.contains(&candidate.branch)
13599            && git::branch_exists(&state.repo, &candidate.branch).await?
13600        {
13601            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13602        }
13603    }
13604    let mut cargo_guards = Vec::new();
13605    let entries = match std::fs::read_dir(state.worktree_root()) {
13606        Ok(entries) => Some(entries),
13607        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13608        Err(error) => return Err(error.into()),
13609    };
13610    for entry in entries.into_iter().flatten() {
13611        let path = entry?.path();
13612        let metadata = std::fs::symlink_metadata(&path)?;
13613        anyhow::ensure!(
13614            metadata.is_dir() && !metadata.file_type().is_symlink(),
13615            "non-directory or symlink recovery source retained: {}",
13616            path.display()
13617        );
13618        if !drop_winner
13619            && winner.is_some_and(|w| {
13620                path.file_name()
13621                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13622            })
13623        {
13624            continue;
13625        }
13626        anyhow::ensure!(
13627            path.join(".git").is_file(),
13628            "unregistered recovery source retained: {}",
13629            path.display()
13630        );
13631        anyhow::ensure!(
13632            git::git(
13633                &path,
13634                &[
13635                    "status",
13636                    "--porcelain",
13637                    "--untracked-files=all",
13638                    "--ignored",
13639                    "--",
13640                    ".",
13641                    ":(exclude)target"
13642                ]
13643            )
13644            .await?
13645            .is_empty(),
13646            "source changes or ignored recovery files retained: {}",
13647            path.display()
13648        );
13649        match std::fs::symlink_metadata(path.join("target")) {
13650            Ok(_) => {
13651                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13652            }
13653            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13654            Err(error) => return Err(error.into()),
13655        }
13656        for checkpoint in [
13657            "rebase-merge",
13658            "rebase-apply",
13659            "sequencer",
13660            "CHERRY_PICK_HEAD",
13661            "MERGE_HEAD",
13662            "REBASE_HEAD",
13663            "REVERT_HEAD",
13664            "BISECT_LOG",
13665            "AUTO_MERGE",
13666            "MERGE_MSG",
13667        ] {
13668            let location = git::git(
13669                &path,
13670                &[
13671                    "rev-parse",
13672                    "--path-format=absolute",
13673                    "--git-path",
13674                    checkpoint,
13675                ],
13676            )
13677            .await?;
13678            anyhow::ensure!(
13679                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13680                if e.kind() == std::io::ErrorKind::NotFound),
13681                "Git recovery checkpoint retained: {checkpoint}"
13682            );
13683        }
13684        heads.insert(git::rev_parse(&path, "HEAD").await?);
13685    }
13686    crate::fold_evidence::preserve(state, home, &heads).await?;
13687
13688    Ok(cargo_guards)
13689}
13690
13691/// Remove `dir` if it exists and has nothing in it.
13692///
13693/// Best-effort and silent by design: a directory that is not empty (a run
13694/// whose winner is still parked there, a stray file some other process left)
13695/// is exactly the case this must refuse, and a directory that is already gone
13696/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13697/// already refuses a non-empty directory, so the emptiness check below is
13698/// belt, not suspenders - it is what keeps this from ever attempting the
13699/// removal in the case that matters, rather than trusting `remove_dir`'s
13700/// error path to have no side effects if it ever changed.
13701fn remove_if_empty(dir: &Path) {
13702    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13703        std::fs::remove_dir(dir).ok();
13704    }
13705}
13706
13707/// Severity of the worst open finding in the last review round, for reporting.
13708pub fn worst_open(state: &RunState) -> Option<Severity> {
13709    state
13710        .reviews
13711        .last()?
13712        .reviews
13713        .iter()
13714        .flat_map(|r| r.findings.iter())
13715        .map(|f| f.severity)
13716        .max()
13717}
13718
13719/// `[worktree] setup` hides its products per worktree, which needs
13720/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13721/// worktrees, like the hook's; every place that runs setup calls this first.
13722async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13723    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13724        git::acquire_worktree_config(repo).await?;
13725        state.enabled_worktree_config = true;
13726    }
13727    Ok(())
13728}
13729
13730#[cfg(test)]
13731mod tests {
13732    #[tokio::test]
13733    async fn launcher_base_fetch_requires_remote_success() {
13734        let tmp = tempfile::tempdir().unwrap();
13735        let repo = tmp.path().join("repo");
13736        let remote = tmp.path().join("remote");
13737        std::fs::create_dir(&repo).unwrap();
13738        std::fs::create_dir(&remote).unwrap();
13739        git::git(&remote, &["init", "--bare", "-b", "main"])
13740            .await
13741            .unwrap();
13742        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13743        git::git(
13744            &repo,
13745            &[
13746                "-c",
13747                "user.name=Fixture",
13748                "-c",
13749                "user.email=fixture@example.invalid",
13750                "commit",
13751                "--allow-empty",
13752                "-m",
13753                "fixture",
13754            ],
13755        )
13756        .await
13757        .unwrap();
13758        git::git(
13759            &repo,
13760            &["remote", "add", "origin", remote.to_str().unwrap()],
13761        )
13762        .await
13763        .unwrap();
13764        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13765        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13766        assert_eq!(
13767            resolve_base(&repo, "main", "origin", None)
13768                .await
13769                .unwrap()
13770                .commit,
13771            head
13772        );
13773        #[cfg(unix)]
13774        {
13775            let policy: crate::config::Coordination = toml::from_str(
13776                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13777            ).unwrap();
13778            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13779            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13780            let held = crate::repo_lock::SeatLease::operation_with(
13781                &repo,
13782                Some(&policy),
13783                &crate::repo_lock::WaitPolicy::default(),
13784            )
13785            .await
13786            .unwrap();
13787            assert!(held.is_some());
13788            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13789            tokio::pin!(fetch);
13790            assert!(
13791                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13792                    .await
13793                    .is_err()
13794            );
13795            assert!(
13796                !repo.join(".git/FETCH_HEAD").exists(),
13797                "no fetch before admission"
13798            );
13799            drop(held);
13800            let fetched = fetch.await.unwrap();
13801            assert_eq!(fetched.commit, head);
13802            assert_eq!(fetched.remote, "origin");
13803            assert_eq!(fetched.branch, "main");
13804        }
13805        // A surviving tracking ref is not successful fetch evidence.
13806        git::git(
13807            &repo,
13808            &[
13809                "remote",
13810                "set-url",
13811                "origin",
13812                tmp.path().join("missing").to_str().unwrap(),
13813            ],
13814        )
13815        .await
13816        .unwrap();
13817        let error = resolve_base(&repo, "main", "origin", None)
13818            .await
13819            .unwrap_err()
13820            .to_string();
13821        assert!(error.contains("cannot read origin/main"));
13822        assert!(error.contains("refusing to branch off"));
13823    }
13824
13825    #[test]
13826    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13827        let leaked = vec!["chore(deps): update a crate".to_owned()];
13828        let own = "fix(daemon): apply a chosen action";
13829        assert!(should_retitle("", own, &leaked));
13830        assert!(should_retitle(
13831            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13832            own,
13833            &leaked
13834        ));
13835        assert!(should_retitle(
13836            "chore: land candidate A of run 1",
13837            own,
13838            &leaked
13839        ));
13840        assert!(should_retitle(
13841            "magi: candidate A (uncommitted work)",
13842            own,
13843            &leaked
13844        ));
13845        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13846        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13847        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13848    }
13849
13850    #[test]
13851    fn pr_merge_plan_creates_adopts_or_stops() {
13852        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13853        assert_eq!(
13854            pr_merge_plan(Ok(land::OpenPr::One {
13855                url: "u".into(),
13856                title: "t".into()
13857            })),
13858            PrPlan::Adopt {
13859                url: "u".into(),
13860                title: "t".into()
13861            }
13862        );
13863        let PrPlan::Stop(many) =
13864            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13865        else {
13866            panic!("many must stop");
13867        };
13868        assert!(many.contains('a') && many.contains('b'));
13869        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13870            panic!("a failed lookup must stop");
13871        };
13872        assert!(err.contains("bad token"));
13873    }
13874
13875    use super::*;
13876    use crate::run::GateStatus;
13877    use std::collections::BTreeMap;
13878    use std::time::Duration;
13879
13880    fn conductor() -> AgentSpec {
13881        AgentSpec {
13882            id: "conductor".to_owned(),
13883            kind: crate::config::AgentKind::Command,
13884            model: None,
13885            command: vec!["true".to_owned()],
13886            extra_args: Vec::new(),
13887            env: BTreeMap::new(),
13888            prompt_delivery: None,
13889            max_turn_secs: None,
13890        }
13891    }
13892
13893    fn spec(id: &str) -> AgentSpec {
13894        AgentSpec {
13895            id: id.to_owned(),
13896            kind: crate::config::AgentKind::Command,
13897            model: None,
13898            command: vec!["true".to_owned()],
13899            extra_args: Vec::new(),
13900            env: BTreeMap::new(),
13901            prompt_delivery: None,
13902            max_turn_secs: None,
13903        }
13904    }
13905
13906    fn ids(xs: &[&str]) -> BTreeSet<String> {
13907        xs.iter().map(|s| (*s).to_owned()).collect()
13908    }
13909
13910    #[test]
13911    fn pick_successor_skips_an_agent_another_seat_holds() {
13912        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13913        let roster = [spec("a"), spec("b"), spec("c")];
13914        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13915        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13916    }
13917
13918    #[test]
13919    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13920        // The other seat started on c but was handed to d; c is free again.
13921        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13922        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13923        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13924        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13925        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13926    }
13927
13928    #[test]
13929    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13930        let roster = [spec("a"), spec("b")];
13931        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13932        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13933        let carried = ids(&["b"]);
13934        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13935        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13936    }
13937
13938    #[test]
13939    fn pick_successor_returns_none_without_any_untried_successor() {
13940        let roster = [spec("a"), spec("b")];
13941        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13942        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13943        let carried = ids(&["a"]);
13944        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13945    }
13946
13947    #[test]
13948    fn pick_successor_rescue_avoids_another_seats_occupant() {
13949        // b is a carried failure and free; a is held by the other seat.
13950        let roster = [spec("a"), spec("b"), spec("c")];
13951        let carried = ids(&["b", "c"]);
13952        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13953        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13954    }
13955
13956    #[test]
13957    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13958        let roster = [spec("a"), spec("b"), spec("c")];
13959        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13960        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13961    }
13962
13963    #[test]
13964    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13965        let roster = [spec("a"), spec("b"), spec("c")];
13966        let failed = ids(&["a", "b", "c"]);
13967        // Rescue looks at the whole roster, once per id, then runs out.
13968        let mut tried = ids(&["b"]);
13969        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13970        assert_eq!(first.id, "a");
13971        tried.insert(first.id.clone());
13972        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13973        assert_eq!(second.id, "c");
13974        tried.insert(second.id.clone());
13975        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13976    }
13977
13978    #[test]
13979    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13980        let roster = [spec("a"), spec("b")];
13981        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13982        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13983    }
13984
13985    #[test]
13986    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13987        let roster = [spec("a"), spec("b"), spec("c")];
13988        let h = SeatHistory {
13989            failed: ids(&["a"]),
13990            last_ok: Some("b".to_owned()),
13991            last_fail: None,
13992        };
13993        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13994        // A last answerer that left the roster, or later failed, is ignored.
13995        let gone = SeatHistory {
13996            last_ok: Some("zzz".to_owned()),
13997            ..h.clone()
13998        };
13999        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
14000        let failed = SeatHistory {
14001            failed: ids(&["a", "b"]),
14002            last_ok: Some("b".to_owned()),
14003            last_fail: None,
14004        };
14005        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
14006    }
14007
14008    #[test]
14009    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
14010        let first = SeatState::new("review-1", "alpha", 7);
14011        let next = handover_seat("review-1", "gamma", 7);
14012        assert_ne!(first.claude_session, next.claude_session);
14013    }
14014
14015    #[test]
14016    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
14017        let mut state = state_with_summary("x", "y");
14018        let a = handover_seat("review-1", "beta", state.next_seat_seed());
14019        let b = handover_seat("review-1", "beta", state.next_seat_seed());
14020        assert_ne!(a.claude_session, b.claude_session);
14021    }
14022
14023    #[test]
14024    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
14025        let roster = [spec("a"), spec("b")];
14026        let h = SeatHistory {
14027            failed: ids(&["a", "b"]),
14028            ..SeatHistory::default()
14029        };
14030        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
14031        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
14032        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
14033    }
14034
14035    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
14036    // fallback loop depends on to terminate: it must walk forward from the
14037    // seat's own position, never restart at the front of the roster, and it
14038    // must never hand back an id already tried, however many times that id
14039    // happens to appear.
14040
14041    #[test]
14042    fn failure_signature_ignores_numbers_and_paths() {
14043        assert_eq!(
14044            failure_signature("exited with Some(2) and no usable output"),
14045            failure_signature("exited with Some(137) and no usable output")
14046        );
14047        assert_eq!(
14048            failure_signature("process exited with no usable output (exit code 2)"),
14049            failure_signature("process exited with no usable output (exit code 137)")
14050        );
14051        for (old, new) in [
14052            (
14053                "exited with Some(2) and no usable output",
14054                "process exited with no usable output (exit code 2)",
14055            ),
14056            (
14057                "exited with None and no usable output",
14058                "process exited with no usable output (exit code unavailable)",
14059            ),
14060        ] {
14061            let previous = FailClass::Other(failure_signature(old));
14062            let current = FailClass::Other(failure_signature(new));
14063            assert_eq!(previous, current, "resumption retains the failure class");
14064            assert!(!should_hand_over(Some(&previous), &current));
14065        }
14066        assert_eq!(
14067            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
14068            failure_signature("cannot open /var/x.txt: denied")
14069        );
14070        assert_ne!(failure_signature("boom"), failure_signature("bang"));
14071    }
14072
14073    #[test]
14074    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
14075        let other = FailClass::Other("x".into());
14076        assert!(should_hand_over(None, &FailClass::Quota));
14077        assert!(should_hand_over(Some(&other), &FailClass::Quota));
14078        assert!(should_hand_over(
14079            Some(&FailClass::Timeout),
14080            &FailClass::Timeout
14081        ));
14082        assert!(should_hand_over(None, &other));
14083        assert!(!should_hand_over(Some(&other), &other));
14084        assert!(should_hand_over(
14085            Some(&other),
14086            &FailClass::Other("y".into())
14087        ));
14088        // A quota or timeout in between ends the run of identical failures.
14089        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14090        assert!(should_hand_over(Some(&FailClass::Quota), &other));
14091    }
14092
14093    #[test]
14094    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14095        let a = SeatState::new("judge-1", "alpha", 7);
14096        let b = handover_seat("judge-1", "beta", 7);
14097        assert_ne!(a.claude_session, b.claude_session);
14098        assert_eq!(b.turns, 0);
14099    }
14100
14101    #[test]
14102    fn a_timeout_is_classified_apart_from_other_failures() {
14103        assert_eq!(
14104            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14105            Some(FailClass::Timeout)
14106        );
14107        assert!(matches!(
14108            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14109            Some(FailClass::Other(_))
14110        ));
14111    }
14112
14113    #[test]
14114    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14115        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14116        let tried = BTreeSet::from(["beta".to_owned()]);
14117        // beta sits at index 1; the next candidate is gamma, never alpha —
14118        // which is very likely a different candidate slot's own agent.
14119        let next = next_untried_in_roster(&roster, 1, &tried);
14120        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14121    }
14122
14123    #[test]
14124    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14125        let roster = vec![spec("alpha"), spec("beta")];
14126        let tried = BTreeSet::from(["beta".to_owned()]);
14127        // beta is the roster's last entry: nothing follows it, and alpha —
14128        // earlier in the roster, almost certainly a different candidate
14129        // slot's own agent — must not be reached by wrapping back to it.
14130        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14131    }
14132
14133    #[test]
14134    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14135        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14136        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14137        // beta (index 1) and gamma (index 2, the only entry after it) have
14138        // both been tried; alpha (index 0) never has, but it comes before
14139        // beta's own position, so there is nothing further for this seat.
14140        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14141    }
14142
14143    #[test]
14144    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14145        let roster = vec![spec("a"), spec("a"), spec("b")];
14146        let tried = BTreeSet::from(["a".to_owned()]);
14147        let next = next_untried_in_roster(&roster, 0, &tried);
14148        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14149    }
14150
14151    #[test]
14152    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14153        let roster = vec![spec("a"), spec("b")];
14154        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14155        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14156    }
14157
14158    #[test]
14159    fn remove_if_empty_only_ever_takes_a_bare_directory() {
14160        let dir = tempfile::tempdir().unwrap();
14161        let bay = dir.path().join("ffff");
14162
14163        // Not there yet: nothing to do, nothing to panic on.
14164        remove_if_empty(&bay);
14165        assert!(!bay.exists());
14166
14167        // Something still inside - the winner's worktree, or a stray file -
14168        // keeps the directory standing.
14169        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14170        remove_if_empty(&bay);
14171        assert!(bay.exists(), "non-empty directory must survive");
14172
14173        // Once the last entry is gone, so is the directory itself.
14174        std::fs::remove_dir(bay.join("cand-A")).unwrap();
14175        remove_if_empty(&bay);
14176        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14177    }
14178
14179    // `round_is_clean` is the exact decision this task fixed: a round with a
14180    // seat that never answered must not read the same as a round every seat
14181    // actually reviewed. These are deterministic and process-free by design —
14182    // the equivalent end-to-end check (a real reviewer timing out under a
14183    // live graph run) is a genuine race against wall-clock contention, and a
14184    // spawn slow enough to blow even a generous budget under a loaded test
14185    // run must not turn this specific regression check flaky.
14186
14187    #[test]
14188    fn a_full_panel_that_found_nothing_is_clean() {
14189        assert!(round_is_clean(
14190            0,
14191            true,
14192            2,
14193            2,
14194            0,
14195            IncompleteReviewPolicy::Block
14196        ));
14197    }
14198
14199    #[test]
14200    fn a_missing_seat_is_never_clean_under_the_default_policy() {
14201        assert!(!round_is_clean(
14202            0,
14203            true,
14204            1,
14205            2,
14206            0,
14207            IncompleteReviewPolicy::Block
14208        ));
14209    }
14210
14211    #[test]
14212    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14213        assert!(!round_is_clean(
14214            1,
14215            true,
14216            1,
14217            2,
14218            0,
14219            IncompleteReviewPolicy::Warn
14220        ));
14221    }
14222
14223    #[test]
14224    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14225        assert!(round_is_clean(
14226            0,
14227            true,
14228            1,
14229            2,
14230            0,
14231            IncompleteReviewPolicy::Warn
14232        ));
14233    }
14234
14235    #[test]
14236    fn a_full_panel_with_an_open_finding_is_not_clean() {
14237        assert!(!round_is_clean(
14238            1,
14239            true,
14240            2,
14241            2,
14242            0,
14243            IncompleteReviewPolicy::Block
14244        ));
14245    }
14246
14247    #[test]
14248    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14249        assert!(!round_is_clean(
14250            0,
14251            false,
14252            2,
14253            2,
14254            0,
14255            IncompleteReviewPolicy::Block
14256        ));
14257    }
14258
14259    // The stall this task closes: under the default `block` policy, a seat
14260    // missing only because it was rate limited must not force a wait for a
14261    // session limit that will not lift by the next round. `round_is_clean`
14262    // is where that quorum carve-out lives; the review loop around it never
14263    // changes what a reviewer's vote or a finding's severity means.
14264
14265    #[test]
14266    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14267        // 1 of 2 answered, and the one missing was quota'd — the exact
14268        // "review-2 rate limited (quota)" shape from the field report.
14269        assert!(round_is_clean(
14270            0,
14271            true,
14272            1,
14273            2,
14274            1,
14275            IncompleteReviewPolicy::Block
14276        ));
14277    }
14278
14279    #[test]
14280    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14281        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14282        // not a quota loss (`quota_missing` stays 0) — worth another try.
14283        assert!(!round_is_clean(
14284            0,
14285            true,
14286            1,
14287            2,
14288            0,
14289            IncompleteReviewPolicy::Block
14290        ));
14291    }
14292
14293    #[test]
14294    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14295        assert!(!round_is_clean(
14296            1,
14297            true,
14298            1,
14299            2,
14300            1,
14301            IncompleteReviewPolicy::Block
14302        ));
14303        assert!(!round_is_clean(
14304            0,
14305            false,
14306            1,
14307            2,
14308            1,
14309            IncompleteReviewPolicy::Block
14310        ));
14311    }
14312
14313    #[test]
14314    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14315        // Every seat quota'd, nobody answered: there is no panel to decide
14316        // on, so this must fall through to the existing block-and-retry
14317        // fallback rather than call an unreviewed patch clean.
14318        assert!(!round_is_clean(
14319            0,
14320            true,
14321            0,
14322            2,
14323            2,
14324            IncompleteReviewPolicy::Block
14325        ));
14326    }
14327
14328    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14329        CommandOutcome {
14330            command: "test".to_owned(),
14331            code,
14332            output_tail: String::new(),
14333            duration_ms: 0,
14334            resource_blocked,
14335        }
14336    }
14337
14338    #[cfg(unix)]
14339    #[tokio::test]
14340    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14341        ask_test_home();
14342        let temp = tempfile::tempdir().unwrap();
14343        let mut runner = runner_at(RunStatus::Gating);
14344        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14345        let commands = vec![
14346            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14347            "touch must-not-run".to_owned(),
14348        ];
14349        let (outcomes, retried, pids) = run_e2e_with_retry(
14350            &mut runner.state,
14351            &shell,
14352            &commands,
14353            temp.path(),
14354            Duration::from_secs(5),
14355            "fixture",
14356        )
14357        .await;
14358        assert_eq!(outcomes.len(), 1);
14359        assert!(outcomes[0].resource_blocked);
14360        assert!(!outcomes[0].build_failed());
14361        assert!(!retried);
14362        assert!(pids.is_empty());
14363        assert!(!temp.path().join("must-not-run").exists());
14364        assert_eq!(
14365            runner.state.resource_block.as_ref().unwrap().kind,
14366            "incremental cache admission"
14367        );
14368    }
14369
14370    #[test]
14371    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14372        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14373        assert!(
14374            !verify_inconclusive(&[outcome(Some(1), false)]),
14375            "an ordinary failure is still evidence about the patch"
14376        );
14377        assert!(verify_inconclusive(&[outcome(None, true)]));
14378        assert!(
14379            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14380            "one inconclusive outcome taints the whole batch"
14381        );
14382        assert!(!verify_inconclusive(&[]));
14383    }
14384
14385    #[tokio::test]
14386    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14387        // Alive for the first two checks, then dead - confirms the loop
14388        // actually re-polls rather than deciding once and sleeping out the
14389        // ceiling regardless.
14390        let calls = std::sync::atomic::AtomicUsize::new(0);
14391        let started = Instant::now();
14392        wait_for_pids_with(
14393            &[123],
14394            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14395            Duration::from_millis(5),
14396            Duration::from_secs(5),
14397        )
14398        .await;
14399        assert!(
14400            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14401            "must keep checking rather than deciding on the first answer"
14402        );
14403        assert!(
14404            started.elapsed() < Duration::from_secs(1),
14405            "must return the moment it is confirmed dead, not wait out the ceiling"
14406        );
14407    }
14408
14409    #[tokio::test]
14410    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14411        let started = Instant::now();
14412        wait_for_pids_with(
14413            &[123],
14414            |_| true, // never reports dead
14415            Duration::from_millis(5),
14416            Duration::from_millis(30),
14417        )
14418        .await;
14419        let elapsed = started.elapsed();
14420        assert!(
14421            elapsed >= Duration::from_millis(30),
14422            "must not give up before its own ceiling: {elapsed:?}"
14423        );
14424        assert!(
14425            elapsed < Duration::from_secs(1),
14426            "must not wait past its own ceiling either: {elapsed:?}"
14427        );
14428    }
14429
14430    #[tokio::test]
14431    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14432        let started = Instant::now();
14433        wait_for_pids_with(
14434            &[],
14435            |_| true,
14436            Duration::from_secs(5),
14437            Duration::from_secs(5),
14438        )
14439        .await;
14440        assert!(
14441            started.elapsed() < Duration::from_millis(200),
14442            "an empty pid list has nothing to confirm"
14443        );
14444    }
14445
14446    // `review_conclusion` is the exact decision the review hand-off task
14447    // fixed: a round budget spent (or a tree that stopped moving) must not
14448    // collapse into `Blocked` regardless of what verification actually
14449    // said. Deterministic and process-free for the same reason the
14450    // `round_is_clean` family above is.
14451    fn review_round(
14452        clean: bool,
14453        blocking: usize,
14454        answered: usize,
14455        expected: usize,
14456        progressed: bool,
14457        e2e_ok: bool,
14458    ) -> ReviewRound {
14459        ReviewRound {
14460            round: 1,
14461            head: "h".to_owned(),
14462            verified_head: None,
14463            verified_at: None,
14464            reviews: Vec::new(),
14465            e2e: vec![CommandOutcome {
14466                command: "test".to_owned(),
14467                code: Some(if e2e_ok { 0 } else { 1 }),
14468                output_tail: String::new(),
14469                duration_ms: 0,
14470                resource_blocked: false,
14471            }],
14472            verify_retried: false,
14473            e2e_deferred: false,
14474            e2e_defer_reason: None,
14475            fix: None,
14476            blocking,
14477            answered,
14478            expected,
14479            clean,
14480            progressed,
14481            vote_split: false,
14482            reconsideration: Vec::new(),
14483            verdict: None,
14484        }
14485    }
14486
14487    #[test]
14488    fn review_conclusion_is_none_when_nothing_has_run() {
14489        assert_eq!(review_conclusion(&[], 3), None);
14490    }
14491
14492    #[test]
14493    fn review_conclusion_is_none_while_rounds_remain() {
14494        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14495        assert_eq!(review_conclusion(&rounds, 3), None);
14496    }
14497
14498    #[test]
14499    fn review_conclusion_is_gating_once_a_round_is_clean() {
14500        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14501        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14502    }
14503
14504    #[test]
14505    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14506        let rounds = vec![
14507            review_round(false, 1, 2, 2, true, true),
14508            review_round(false, 1, 2, 2, true, true),
14509        ];
14510        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14511    }
14512
14513    #[test]
14514    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14515        let rounds = vec![
14516            review_round(false, 1, 2, 2, true, true),
14517            review_round(false, 1, 2, 2, true, false),
14518        ];
14519        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14520    }
14521
14522    #[test]
14523    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14524        // Magi never got a command to run against this round's own head — a
14525        // resource-blocked attempt, not a red one — so this must never
14526        // settle on `Blocked` the way a genuine e2e failure would. `None`
14527        // here is what tells `Runner::review_loop` to retry the check
14528        // itself rather than trust this cheap recomputation with a verdict
14529        // it cannot actually produce.
14530        let mut blocked = review_round(false, 1, 2, 2, true, false);
14531        blocked.e2e[0].resource_blocked = true;
14532        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14533        assert_eq!(review_conclusion(&rounds, 2), None);
14534    }
14535
14536    #[test]
14537    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14538        // Missing input, not a verified tree — never a hand-off candidate.
14539        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14540        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14541    }
14542
14543    #[test]
14544    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14545        let rounds = vec![
14546            review_round(false, 1, 2, 2, false, true),
14547            review_round(false, 1, 2, 2, false, true),
14548        ];
14549        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14550    }
14551
14552    fn secs(n: u64) -> Duration {
14553        Duration::from_secs(n)
14554    }
14555
14556    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14557    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14558    #[tokio::test]
14559    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14560        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14561        if std::env::var_os(CHILD).is_none() {
14562            // Other tests reconcile every run in the process-global home,
14563            // briefly taking owner locks. This ancestry test expects an idle
14564            // predecessor, so give it its own process and pinned test home.
14565            let output = std::process::Command::new(std::env::current_exe().unwrap())
14566                .args([
14567                    "--exact",
14568                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14569                    "--nocapture",
14570                ])
14571                .env(CHILD, "1")
14572                .env_remove("MAGI_WORKER_HOME")
14573                .env_remove("MAGI_WORKER_ID")
14574                .output()
14575                .unwrap();
14576            assert!(
14577                output.status.success()
14578                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14579                "{}\n{}",
14580                String::from_utf8_lossy(&output.stdout),
14581                String::from_utf8_lossy(&output.stderr)
14582            );
14583            return;
14584        }
14585        ask_test_home();
14586        let tmp = tempfile::tempdir().unwrap();
14587        let repo = tmp.path().join("repo");
14588        std::fs::create_dir_all(&repo).unwrap();
14589        init_repo(&repo);
14590        let g = |args: &[&str]| {
14591            let out = std::process::Command::new("git")
14592                .args(args)
14593                .current_dir(&repo)
14594                .quiet()
14595                .output()
14596                .unwrap();
14597            assert!(
14598                out.status.success(),
14599                "{}",
14600                String::from_utf8_lossy(&out.stderr)
14601            );
14602        };
14603        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14604        g(&["checkout", "-b", "historical"]);
14605        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14606        g(&["add", "."]);
14607        g(&["commit", "-m", "old alternative"]);
14608        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14609        g(&["checkout", "-b", "magi/task-selected", &base]);
14610        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14611        g(&["add", "."]);
14612        g(&["commit", "-m", "measurement infrastructure"]);
14613        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14614        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14615        g(&["add", "."]);
14616        g(&["commit", "-m", "dependent implementation"]);
14617        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14618        g(&["checkout", "main"]);
14619        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14620        g(&["add", "."]);
14621        g(&["commit", "-m", "unrelated base change"]);
14622        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14623        let origin = tmp.path().join("origin.git");
14624        g(&[
14625            "clone",
14626            "--bare",
14627            repo.to_str().unwrap(),
14628            origin.to_str().unwrap(),
14629        ]);
14630        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14631        g(&["fetch", "origin"]);
14632        let selection = crate::pr_recovery::Selection {
14633            url: "https://example.test/project/pull/1".into(),
14634            branch: "magi/task-selected".into(),
14635            head: head.clone(),
14636            source_head: None,
14637            source_run: None,
14638            source_proof: None,
14639            disposition: Some(crate::pr_recovery::Disposition::Review),
14640            instruction: "preserve measured behavior".into(),
14641            approved_at: jiff::Timestamp::now(),
14642        };
14643        let mut template = runner_at(RunStatus::Reviewing);
14644        template.state.repo = repo.clone();
14645        template.state.base_commit = current.clone();
14646        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14647        template.state.origin = Some(crate::run::Origin::queue("selected"));
14648        template.state.pr_head = Some(selection.branch.clone());
14649        template.state.pr_recovery = Some(selection.clone());
14650        let mut runner =
14651            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14652                .await
14653                .unwrap();
14654        let wt = runner.state.winner().unwrap().worktree.clone();
14655        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14656        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14657        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14658        assert!(runner.state.seeds.is_empty());
14659        assert_eq!(runner.state.instruction, "preserve measured behavior");
14660        runner.sync_to_base().await.unwrap();
14661        assert_eq!(
14662            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14663            "prerequisite"
14664        );
14665        assert_eq!(
14666            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14667            "measured implementation"
14668        );
14669        assert_eq!(
14670            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14671            "current main change"
14672        );
14673        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14674        assert_eq!(
14675            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14676            head
14677        );
14678
14679        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14680        let mut partial = runner_at(RunStatus::Reviewing);
14681        partial.state.repo = repo.clone();
14682        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14683        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14684            run: Some("exit 1".into()),
14685            ..Default::default()
14686        }];
14687        let selection = runner.state.pr_recovery.clone().unwrap();
14688        partial.state.pr_recovery = Some(selection.clone());
14689        partial.state.pr_head = Some(selection.branch.clone());
14690        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14691        let id = partial.state.id.clone();
14692        let error = match Runner::open_selected(
14693            &repo,
14694            partial.state,
14695            partial.roles,
14696            1,
14697            runner.state.base_commit.clone(),
14698            selection,
14699        )
14700        .await
14701        {
14702            Ok(_) => panic!("setup must fail"),
14703            Err(e) => e,
14704        };
14705        assert_eq!(
14706            error
14707                .downcast_ref::<crate::run::StartupFailure>()
14708                .unwrap()
14709                .run,
14710            id
14711        );
14712        let failed = RunState::load(&id).unwrap();
14713        assert_eq!(failed.status, RunStatus::Failed);
14714        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14715        assert!(Runner::resume(&id).is_err());
14716        runner.state.status = RunStatus::Ready;
14717        runner.state.driver_exited = true;
14718        runner.state.save().unwrap();
14719        let mut retry = runner_at(RunStatus::Reviewing);
14720        retry.state.repo = repo.clone();
14721        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14722        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14723        reselected.approved_at = jiff::Timestamp::now();
14724        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14725        retry.state.pr_head = Some(reselected.branch.clone());
14726        retry.state.pr_recovery = Some(reselected.clone());
14727        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14728            .await
14729            .unwrap();
14730        let restarted = Runner::open_selected(
14731            &repo,
14732            retry.state,
14733            retry.roles,
14734            1,
14735            runner.state.base_commit.clone(),
14736            reselected,
14737        )
14738        .await
14739        .unwrap();
14740        assert_eq!(
14741            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14742                .await
14743                .unwrap(),
14744            head
14745        );
14746        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14747        assert_eq!(
14748            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14749            head
14750        );
14751    }
14752
14753    #[tokio::test]
14754    #[cfg(unix)]
14755    async fn fallback_and_interrupted_restoration_require_coordination() {
14756        use std::os::fd::AsRawFd as _;
14757        ask_test_home();
14758        let tmp = tempfile::tempdir().unwrap();
14759        let mut runner = registered_runner(tmp.path()).await;
14760        runner.state.config.graph.timeout_resource_wait = 1;
14761        let winner = runner.state.candidates[0].clone();
14762        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14763        let leftover = winner.worktree.join("leftover");
14764        std::fs::write(&leftover, "untrusted edits").unwrap();
14765        let next = tmp.path().join("next-fixer");
14766        let mut failed = spec("first");
14767        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14768        let mut successor = spec("second");
14769        successor.command = vec![
14770            "sh".into(),
14771            "-c".into(),
14772            format!("touch '{}'; echo done", next.display()),
14773        ];
14774        let attempts = vec![
14775            (failed.clone(), "first".into()),
14776            (successor.clone(), "second".into()),
14777        ];
14778        let lock_path = runner.state.repo.join(".git/updater.lock");
14779        let external = std::fs::File::create(&lock_path).unwrap();
14780        assert_eq!(
14781            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14782            0
14783        );
14784        let artifacts = tmp.path().join("artifacts");
14785        let build = |spec, seat| SeatJob {
14786            spec,
14787            seat,
14788            cwd: winner.worktree.clone(),
14789            prompt: "fixture".into(),
14790            timeout: Duration::from_secs(10),
14791            allow_write: true,
14792            sessions: false,
14793            artifacts: artifacts.clone(),
14794            stem: "fix".into(),
14795            handover: None,
14796        };
14797        let (_, _, out) = runner
14798            .ask_fixer_among(
14799                attempts.clone(),
14800                "graft",
14801                None,
14802                Some((&winner.worktree, &head)),
14803                build,
14804            )
14805            .await;
14806        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14807        assert!(leftover.exists(), "contention must prevent clean");
14808        assert!(!next.exists());
14809        assert!(runner.state.handovers.is_empty());
14810        // Early interrupted-repair rollback uses the same protected route.
14811        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14812            from_run: "earlier".into(),
14813            generation: 1,
14814            mode: crate::recovery::Mode::VerifyRepair,
14815            reasons: Vec::new(),
14816            reviewed_head: None,
14817            rounds: Vec::new(),
14818            open_findings: Vec::new(),
14819            evidence: None,
14820            criteria_inherited: true,
14821            repair: crate::recovery::Repair::Started {
14822                before: head.clone(),
14823            },
14824        });
14825        assert!(
14826            runner
14827                .recovery_repair(&winner)
14828                .await
14829                .unwrap_err()
14830                .is::<crate::repo_lock::ResourceBlocked>()
14831        );
14832        assert!(matches!(
14833            runner.state.recovery.as_ref().unwrap().repair,
14834            crate::recovery::Repair::Started { .. }
14835        ));
14836        assert!(leftover.exists());
14837        drop(external);
14838        runner.recovery_repair(&winner).await.unwrap();
14839        assert!(!leftover.exists());
14840        assert!(matches!(
14841            runner.state.recovery.as_ref().unwrap().repair,
14842            crate::recovery::Repair::Done { .. }
14843        ));
14844        // A hard lock-open failure also leaves HEAD and untracked work intact.
14845        std::fs::remove_file(&lock_path).unwrap();
14846        std::fs::create_dir(&lock_path).unwrap();
14847        std::fs::write(&leftover, "untrusted edits").unwrap();
14848        let (_, _, out) = runner
14849            .ask_fixer_among(
14850                attempts,
14851                "graft",
14852                None,
14853                Some((&winner.worktree, &head)),
14854                build,
14855            )
14856            .await;
14857        assert!(matches!(out, AgentOutcome::Failed(_)));
14858        assert!(leftover.exists());
14859        assert!(!next.exists());
14860        assert_eq!(
14861            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14862            head
14863        );
14864    }
14865
14866    #[cfg(unix)]
14867    async fn registered_runner(root: &Path) -> Runner {
14868        repository_runner(root, crate::config::test_coordination()).await
14869    }
14870
14871    async fn repository_runner(
14872        root: &Path,
14873        coordination: Option<crate::config::Coordination>,
14874    ) -> Runner {
14875        ask_test_home();
14876        let repo = root.join("repo");
14877        std::fs::create_dir_all(&repo).unwrap();
14878        init_repo(&repo);
14879        if coordination.is_some() {
14880            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14881        }
14882        let mut runner = runner_at(RunStatus::Prep);
14883        runner.state.repo = repo.clone();
14884        runner.state.config.coordination = coordination;
14885        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14886        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14887        runner.state.config.graph.advise = false;
14888        runner.roles.implementers = vec![spec("implement")];
14889        runner.roles.judges = vec![spec("judge")];
14890        runner.prep().await.unwrap();
14891        runner
14892    }
14893
14894    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14895
14896    #[tokio::test]
14897    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14898        let tmp = tempfile::tempdir().unwrap();
14899        let mut runner = repository_runner(tmp.path(), None).await;
14900        let prompts = runner.state.config.prompts.clone();
14901        let run = runner.state.id.clone();
14902        let ctx = WaveCtx {
14903            carry_seats: false,
14904            pauses: None,
14905            run: &run,
14906            node: "advise",
14907            prompts: &prompts,
14908            cache: None,
14909            round: None,
14910        };
14911        let jobs: Vec<_> = ["alpha", "beta"]
14912            .into_iter()
14913            .map(|id| SeatJob {
14914                spec: scripted(id, AUTH_OUTAGE),
14915                seat: SeatState::new(id, id, 7),
14916                prompt: "propose".into(),
14917                cwd: runner.state.candidates[0].worktree.clone(),
14918                timeout: Duration::from_secs(10),
14919                allow_write: false,
14920                sessions: false,
14921                artifacts: agent::artifacts_dir(&runner.state.dir()),
14922                stem: id.into(),
14923                handover: None,
14924            })
14925            .collect();
14926        let mut losses = Vec::new();
14927        let results = ask_wave_with(
14928            jobs.clone(),
14929            runner.sem.clone(),
14930            2,
14931            &[],
14932            &ctx,
14933            &mut losses,
14934            &mut runner.state,
14935            &|_: &str| Ok(()),
14936        )
14937        .await;
14938        assert!(
14939            results
14940                .iter()
14941                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14942        );
14943        assert_eq!(runner.state.quota.len(), 2);
14944        assert!(
14945            runner
14946                .state
14947                .quota
14948                .iter()
14949                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14950        );
14951        assert!(!runner.state.advise_attempted);
14952        runner.stop_for_resource().unwrap();
14953        assert_eq!(runner.state.status, RunStatus::Blocked);
14954        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14955        assert_eq!(
14956            runner.state.resource_block.as_ref().unwrap().kind,
14957            "agent authentication"
14958        );
14959        // A machine loss followed by an ordinary failure is not evidence
14960        // that the whole node was unavailable, and must not earn a refund.
14961        runner.state.resource_block = None;
14962        runner.state.quota.clear();
14963        let mut mixed = jobs;
14964        mixed[1].spec.command = vec!["false".into()];
14965        let results = ask_wave_with(
14966            mixed,
14967            runner.sem.clone(),
14968            0,
14969            &[],
14970            &ctx,
14971            &mut losses,
14972            &mut runner.state,
14973            &|_: &str| Ok(()),
14974        )
14975        .await;
14976        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14977        assert!(runner.state.resource_block.is_none());
14978        assert_eq!(losses.len(), 1);
14979    }
14980
14981    #[tokio::test]
14982    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14983        let tmp = tempfile::tempdir().unwrap();
14984        let mut runner = repository_runner(tmp.path(), None).await;
14985        let implementer = scripted("implement", AUTH_OUTAGE);
14986        runner.roles.implementers = vec![implementer.clone()];
14987        runner.state.config.agents = vec![implementer];
14988        runner.implement().await.unwrap();
14989        assert_eq!(runner.state.status, RunStatus::Blocked);
14990        assert_eq!(runner.state.quota.len(), 1);
14991        assert_eq!(
14992            runner.state.quota[0].cause,
14993            agent::QuotaCause::Authentication
14994        );
14995        let candidate = &runner.state.candidates[0];
14996        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14997        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14998    }
14999
15000    #[tokio::test]
15001    #[cfg(unix)]
15002    async fn parked_implementer_persists_completed_work() {
15003        for registered in [false, true] {
15004            let tmp = tempfile::tempdir().unwrap();
15005            let mut runner = registered_runner(tmp.path()).await;
15006            if !registered {
15007                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
15008            }
15009            let marker = tmp.path().join("started");
15010            let release = tmp.path().join("release");
15011            let mut implementer = spec("implement");
15012            implementer.command = vec![
15013                "sh".into(),
15014                "-c".into(),
15015                format!(
15016                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
15017                    marker.display(),
15018                    release.display()
15019                ),
15020            ];
15021            runner.roles.implementers = vec![implementer.clone()];
15022            runner.state.config.agents = vec![implementer];
15023            let pause = runner.pause.clone();
15024            let (result, ()) = tokio::join!(runner.implement(), async {
15025                tokio::time::timeout(Duration::from_secs(10), async {
15026                    while !marker.exists() {
15027                        tokio::time::sleep(Duration::from_millis(10)).await;
15028                    }
15029                })
15030                .await
15031                .unwrap();
15032                pause.park();
15033                std::fs::write(&release, "go").unwrap();
15034            });
15035            result.unwrap();
15036            let candidate = &runner.state.candidates[0];
15037            assert!(candidate.commits > 0, "{candidate:?}");
15038            assert!(candidate.summary.contains("saved-result"));
15039            assert_eq!(candidate.files, 1);
15040            let before = candidate.commits;
15041            runner.implement().await.unwrap();
15042            assert_eq!(runner.state.candidates[0].commits, before);
15043        }
15044    }
15045
15046    #[tokio::test]
15047    #[cfg(unix)]
15048    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
15049        let tmp = tempfile::tempdir().unwrap();
15050        let mut runner = registered_runner(tmp.path()).await;
15051        let winner = runner.state.candidates[0].clone();
15052        let cache = tmp.path().join("cache");
15053        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
15054        runner.state.config.graph.timeout_resource_wait = 0;
15055        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
15056        let crate::cache::AcquireOutcome::Acquired(held) =
15057            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
15058        else {
15059            panic!("cache must be free")
15060        };
15061        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15062            from_run: "earlier".into(),
15063            generation: 1,
15064            mode: crate::recovery::Mode::VerifyRepair,
15065            reasons: Vec::new(),
15066            reviewed_head: None,
15067            rounds: Vec::new(),
15068            open_findings: Vec::new(),
15069            evidence: None,
15070            criteria_inherited: true,
15071            repair: crate::recovery::Repair::NotStarted,
15072        });
15073        runner.recovery_repair(&winner).await.unwrap();
15074        assert!(matches!(
15075            runner.state.recovery.as_ref().unwrap().repair,
15076            crate::recovery::Repair::WaitingForRestore { .. }
15077        ));
15078        drop(held);
15079        runner.state.resource_block = None;
15080        runner.state.config.graph.timeout_resource_wait = 1;
15081        runner.recovery_repair(&winner).await.unwrap();
15082        assert!(matches!(
15083            runner.state.recovery.as_ref().unwrap().repair,
15084            crate::recovery::Repair::Done { .. }
15085        ));
15086        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15087    }
15088
15089    #[tokio::test]
15090    #[cfg(unix)]
15091    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15092        use std::os::fd::AsRawFd as _;
15093        for graft in [false, true] {
15094            let tmp = tempfile::tempdir().unwrap();
15095            let mut runner = registered_runner(tmp.path()).await;
15096            runner.state.config.graph.timeout_resource_wait = 1;
15097            let winner = runner.state.candidates[0].clone();
15098            let marker = tmp.path().join("fixer-started");
15099            let mut fixer = spec("fixer");
15100            fixer.command = vec![
15101                "sh".into(),
15102                "-c".into(),
15103                format!("echo started >> '{}'; echo done", marker.display()),
15104            ];
15105            runner.roles.fixer = Some(vec![fixer]);
15106            if graft {
15107                let mut loser = winner.clone();
15108                loser.label = 'B';
15109                loser.index = 1;
15110                runner.state.candidates.push(loser);
15111                runner.state.config.graph.graft_runner_up = true;
15112                runner.state.tally = Some(Tally {
15113                    first_choice: BTreeMap::new(),
15114                    borda: BTreeMap::new(),
15115                    winner: winner.label,
15116                    rankings: 1,
15117                    unanimous_initial: true,
15118                    deliberated: false,
15119                    changed_votes: 0,
15120                    unanimous_final: true,
15121                    tie_break: None,
15122                    judges: 1,
15123                    present: 1,
15124                    quorum: 1,
15125                    met_quorum: true,
15126                    uncontested: None,
15127                });
15128                runner.state.judgements.push(crate::run::Judgement {
15129                    judge: 1,
15130                    seat: "judge-1".into(),
15131                    agent: "judge".into(),
15132                    ranking: vec![winner.label, 'B'],
15133                    reasons: BTreeMap::new(),
15134                    confidence: None,
15135                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15136                    order: vec![0, 1],
15137                    failed: None,
15138                    duration_ms: 0,
15139                });
15140            } else {
15141                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15142                    from_run: "earlier".into(),
15143                    generation: 1,
15144                    mode: crate::recovery::Mode::VerifyRepair,
15145                    reasons: Vec::new(),
15146                    reviewed_head: None,
15147                    rounds: Vec::new(),
15148                    open_findings: Vec::new(),
15149                    evidence: None,
15150                    criteria_inherited: true,
15151                    repair: crate::recovery::Repair::NotStarted,
15152                });
15153            }
15154            let reader = crate::repo_lock::SeatLease::for_checkout(
15155                &winner.worktree,
15156                false,
15157                crate::config::test_coordination().as_ref(),
15158            )
15159            .await
15160            .unwrap();
15161            let pause = runner.pause.clone();
15162            let (result, ()) = tokio::join!(
15163                async {
15164                    if graft {
15165                        runner.graft().await
15166                    } else {
15167                        runner.recovery_repair(&winner).await
15168                    }
15169                },
15170                async {
15171                    tokio::time::sleep(Duration::from_millis(100)).await;
15172                    pause.park();
15173                }
15174            );
15175            result.unwrap();
15176            assert!(!marker.exists(), "a blocked seat must not spawn");
15177            // Exercise the persisted state, as a binary replacement/resume does.
15178            runner.state = RunState::load(&runner.state.id).unwrap();
15179            if graft {
15180                assert_eq!(
15181                    runner.state.graft.as_ref().unwrap().outcome,
15182                    GraftOutcome::WaitingForRestore
15183                );
15184            } else {
15185                assert!(matches!(
15186                    runner.state.recovery.as_ref().unwrap().repair,
15187                    crate::recovery::Repair::WaitingForRestore { .. }
15188                ));
15189            }
15190            drop(reader);
15191            runner.pause = Pause::new();
15192            runner.state.resource_block = None;
15193            // Deferred restoration must still respect an external lock owner.
15194            let external =
15195                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15196            assert_eq!(
15197                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15198                0
15199            );
15200            let leftover = winner.worktree.join("leftover");
15201            std::fs::write(&leftover, "earlier fallback edits").unwrap();
15202            let result = if graft {
15203                runner.graft().await
15204            } else {
15205                runner.recovery_repair(&winner).await
15206            };
15207            assert!(
15208                result
15209                    .unwrap_err()
15210                    .is::<crate::repo_lock::ResourceBlocked>()
15211            );
15212            assert!(leftover.exists());
15213            assert!(!marker.exists());
15214            drop(external);
15215            runner.state.resource_block = None;
15216            let result = if graft {
15217                runner.graft().await
15218            } else {
15219                runner.recovery_repair(&winner).await
15220            };
15221            result.unwrap();
15222            assert!(!leftover.exists(), "restore must precede retry");
15223            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15224            if graft {
15225                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15226            } else {
15227                assert!(matches!(
15228                    runner.state.recovery.as_ref().unwrap().repair,
15229                    crate::recovery::Repair::Done { .. }
15230                ));
15231            }
15232        }
15233    }
15234
15235    #[tokio::test]
15236    #[cfg(unix)]
15237    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15238        use std::os::fd::AsRawFd as _;
15239        let tmp = tempfile::tempdir().unwrap();
15240        let mut runner = registered_runner(tmp.path()).await;
15241        runner.state.config.graph.timeout_resource_wait = 1;
15242        let winner = runner.state.candidates[0].clone();
15243        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15244        runner.state.tally = Some(Tally {
15245            first_choice: BTreeMap::new(),
15246            borda: BTreeMap::new(),
15247            winner: winner.label,
15248            rankings: 0,
15249            unanimous_initial: false,
15250            deliberated: false,
15251            changed_votes: 0,
15252            unanimous_final: false,
15253            tie_break: None,
15254            judges: 0,
15255            present: 0,
15256            quorum: 0,
15257            met_quorum: true,
15258            uncontested: Some("fixture".into()),
15259        });
15260        runner.state.graft = Some(GraftRecord {
15261            pre: pre.clone(),
15262            post: None,
15263            lines: None,
15264            limit: None,
15265            agent: None,
15266            outcome: GraftOutcome::Running,
15267            requested: Vec::new(),
15268            adoption: None,
15269            adoption_error: None,
15270        });
15271        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15272        git::commit_all(&winner.worktree, "unfinished graft")
15273            .await
15274            .unwrap();
15275        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15276        let leftover = winner.worktree.join("leftover");
15277        std::fs::write(&leftover, "untrusted edits").unwrap();
15278        let path = runner.state.repo.join(".git/updater.lock");
15279        let external = std::fs::File::create(&path).unwrap();
15280        assert_eq!(
15281            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15282            0
15283        );
15284        assert!(
15285            runner
15286                .graft()
15287                .await
15288                .unwrap_err()
15289                .is::<crate::repo_lock::ResourceBlocked>()
15290        );
15291        assert!(matches!(
15292            runner.state.graft.as_ref().unwrap().outcome,
15293            GraftOutcome::Running
15294        ));
15295        assert_eq!(
15296            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15297            unfinished
15298        );
15299        assert!(leftover.exists());
15300        drop(external);
15301        std::fs::remove_file(&path).unwrap();
15302        std::fs::create_dir(&path).unwrap();
15303        assert!(runner.graft().await.is_err());
15304        assert!(matches!(
15305            runner.state.graft.as_ref().unwrap().outcome,
15306            GraftOutcome::Running
15307        ));
15308        assert_eq!(
15309            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15310            unfinished
15311        );
15312        assert!(leftover.exists());
15313        std::fs::remove_dir(&path).unwrap();
15314        runner.graft().await.unwrap();
15315        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15316        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15317        assert!(!leftover.exists());
15318    }
15319
15320    #[tokio::test]
15321    #[cfg(unix)]
15322    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15323        use std::os::fd::AsRawFd as _;
15324        let tmp = tempfile::tempdir().unwrap();
15325        let mut runner = registered_runner(tmp.path()).await;
15326        runner.state.prep_complete = Some(false);
15327        runner.state.config.graph.timeout_resource_wait = 1;
15328        runner.state.save().unwrap();
15329        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15330        assert_eq!(
15331            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15332            0
15333        );
15334        runner.execute().await.unwrap();
15335        let disk = RunState::load(&runner.state.id).unwrap();
15336        assert!(disk.driver_exited);
15337        assert_eq!(disk.driver_pid, None);
15338        assert_eq!(runner.state.status, RunStatus::Blocked);
15339        assert_eq!(
15340            runner.state.resource_block.as_ref().unwrap().kind,
15341            "repository coordination"
15342        );
15343        assert!(runner.state.prep_failure.is_none());
15344        assert!(runner.state.seats.is_empty());
15345        runner.pause.park();
15346        // Include headroom for Git subprocesses under parallel test load;
15347        // the repository contention deadline itself remains one second.
15348        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15349            .await
15350            .unwrap()
15351            .unwrap();
15352        assert!(runner.state.parked);
15353        assert!(runner.state.prep_failure.is_none());
15354        let disk = RunState::load(&runner.state.id).unwrap();
15355        assert!(disk.driver_exited);
15356        assert_eq!(disk.driver_pid, None);
15357    }
15358
15359    #[tokio::test]
15360    #[cfg(unix)]
15361    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15362        use std::os::fd::AsRawFd as _;
15363        let tmp = tempfile::tempdir().unwrap();
15364        let mut runner = registered_runner(tmp.path()).await;
15365        runner.state.prep_complete = Some(false);
15366        runner.state.config.graph.timeout_resource_wait = 30;
15367        runner.state.save().unwrap();
15368        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15369        assert_eq!(
15370            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15371            0
15372        );
15373        assert!(
15374            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15375                .await
15376                .is_err()
15377        );
15378        let disk = RunState::load(&runner.state.id).unwrap();
15379        assert!(disk.driver_exited);
15380        assert_eq!(disk.driver_pid, None);
15381        assert!(disk.status.resumable());
15382        assert!(runner.state.driver_exited);
15383        assert_eq!(runner.state.driver_pid, None);
15384        runner.state.save().unwrap();
15385        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15386    }
15387
15388    #[tokio::test]
15389    #[cfg(unix)]
15390    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15391        let tmp = tempfile::tempdir().unwrap();
15392        let mut runner = registered_runner(tmp.path()).await;
15393        let winner = runner.state.candidates[0].clone();
15394        let cache = tmp.path().join("cache");
15395        let started = tmp.path().join("pre-gate-started");
15396        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15397        runner.state.config.verify.gate =
15398            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15399        let home = crate::run::home();
15400        let owner =
15401            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15402        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15403            crate::cache::AcquireOutcome::Acquired(g) => g,
15404            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15405        };
15406        let mut gate = Box::pin(runner.run_gate(&winner));
15407        assert!(
15408            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15409                .await
15410                .is_err()
15411        );
15412        assert!(!started.exists(), "pre-gate must wait for the cache too");
15413        // Simulate the wave that already owns the cache entering invoke.
15414        // It must obtain the editor reservation and finish without waiting
15415        // for the gate that is queued behind its cache.
15416        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15417        let editor = crate::repo_lock::SeatLease::checkout_with(
15418            &winner.worktree,
15419            true,
15420            crate::config::test_coordination().as_ref(),
15421            &wait,
15422        )
15423        .await
15424        .unwrap();
15425        drop(editor);
15426        guard.release();
15427        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15428            .await
15429            .unwrap()
15430            .unwrap();
15431        drop(gate);
15432        assert!(outcomes.iter().all(CommandOutcome::ok));
15433        assert!(started.exists());
15434    }
15435
15436    #[tokio::test]
15437    #[cfg(unix)]
15438    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15439        use std::os::fd::AsRawFd as _;
15440
15441        ask_test_home();
15442        let tmp = tempfile::tempdir().unwrap();
15443        let repo = tmp.path().join("repo");
15444        std::fs::create_dir_all(&repo).unwrap();
15445        init_repo(&repo);
15446        let common = repo.join(".git");
15447        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15448        let mut runner = runner_at(RunStatus::Prep);
15449        runner.state.repo = repo.clone();
15450        runner.state.config.coordination = crate::config::test_coordination();
15451        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15452        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15453        runner.state.config.graph.advise = false;
15454        runner.roles.implementers = vec![spec("implement")];
15455        runner.roles.judges = vec![spec("judge")];
15456        runner.prep().await.unwrap();
15457        let winner = runner.state.candidates[0].clone();
15458        // Emulate edits already left by a completed seat. The fixture CLI
15459        // only reports completion; it performs no repository operations.
15460        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15461        let completed = tmp.path().join("seat-completed");
15462        runner.roles.implementers[0].command = vec![
15463            "sh".into(),
15464            "-c".into(),
15465            format!("touch '{}'; echo completed", completed.display()),
15466        ];
15467        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15468        assert_eq!(
15469            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15470            0
15471        );
15472        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15473        let mut implement = Box::pin(runner.implement());
15474        tokio::select! {
15475            result = &mut implement => panic!("rescue must wait: {result:?}"),
15476            result = tokio::time::timeout(Duration::from_secs(10), async {
15477                while !completed.exists() {
15478                    tokio::time::sleep(Duration::from_millis(20)).await;
15479                }
15480            }) => result.unwrap(),
15481        }
15482        assert!(
15483            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15484                .await
15485                .is_err()
15486        );
15487        assert_eq!(
15488            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15489            before
15490        );
15491        assert_eq!(
15492            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15493            0
15494        );
15495        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15496            .await
15497            .unwrap()
15498            .unwrap();
15499        drop(implement);
15500        assert_ne!(
15501            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15502            before
15503        );
15504        assert!(git::is_clean(&winner.worktree).await.unwrap());
15505
15506        let started = tmp.path().join("pre-gate-started");
15507        let proceed = tmp.path().join("pre-gate-proceed");
15508        let checked = tmp.path().join("gate-ran");
15509        runner.state.config.verify.pre_gate = vec![format!(
15510            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15511            started.display(),
15512            proceed.display()
15513        )];
15514        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15515        assert_eq!(
15516            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15517            0
15518        );
15519        let mut gate = Box::pin(runner.run_gate(&winner));
15520        assert!(
15521            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15522                .await
15523                .is_err()
15524        );
15525        assert!(!started.exists());
15526        assert!(!checked.exists());
15527        assert!(git::is_clean(&winner.worktree).await.unwrap());
15528        assert_eq!(
15529            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15530            0
15531        );
15532        tokio::select! {
15533            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15534            result = tokio::time::timeout(Duration::from_secs(10), async {
15535                while !started.exists() {
15536                    tokio::time::sleep(Duration::from_millis(20)).await;
15537                }
15538            }) => result.unwrap(),
15539        }
15540        assert_eq!(
15541            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15542            -1,
15543            "mechanical edits must hold the real lock"
15544        );
15545        std::fs::write(proceed, "ready").unwrap();
15546        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15547            .await
15548            .unwrap()
15549            .unwrap();
15550        drop(gate);
15551        assert!(outcomes.iter().all(CommandOutcome::ok));
15552        assert!(checked.exists());
15553        assert!(runner.state.pre_gate_commit.is_some());
15554        assert!(git::is_clean(&winner.worktree).await.unwrap());
15555        assert_eq!(
15556            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15557            0,
15558            "gate completion must release coordination"
15559        );
15560        drop(outsider);
15561        std::fs::remove_file(common.join("updater.lock")).unwrap();
15562        std::fs::create_dir(common.join("updater.lock")).unwrap();
15563        std::fs::remove_file(&started).unwrap();
15564        std::fs::remove_file(&checked).unwrap();
15565        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15566        let error = runner.run_gate(&winner).await.unwrap_err();
15567        assert!(
15568            error
15569                .to_string()
15570                .contains("acquire mandatory repository lock")
15571        );
15572        assert!(
15573            !started.exists(),
15574            "failed acquisition must prevent pre-gate"
15575        );
15576        assert!(!checked.exists(), "failed acquisition must prevent gate");
15577        assert!(git::is_clean(&winner.worktree).await.unwrap());
15578        assert_eq!(
15579            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15580            head
15581        );
15582    }
15583
15584    #[tokio::test]
15585    #[cfg(unix)]
15586    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15587        use std::os::fd::AsRawFd as _;
15588        use std::time::Duration;
15589
15590        ask_test_home();
15591        let tmp = tempfile::tempdir().unwrap();
15592        let repo = tmp.path().join("repo");
15593        std::fs::create_dir_all(&repo).unwrap();
15594        init_repo(&repo);
15595        let common = repo.join(".git");
15596        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15597        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15598        assert_eq!(
15599            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15600            0
15601        );
15602        let mut runner = runner_at(RunStatus::Prep);
15603        runner.state.repo = repo.clone();
15604        runner.state.config.coordination = crate::config::test_coordination();
15605        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15606        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15607        runner.state.config.graph.advise = true;
15608        runner.state.config.graph.advisors = 1;
15609        runner.roles.implementers = vec![spec("implement")];
15610        runner.roles.judges = vec![spec("judge")];
15611        let root = runner.state.worktree_root();
15612        let started = tmp.path().join("setup-started");
15613        let proceed = tmp.path().join("setup-proceed");
15614        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15615            run: Some(format!(
15616                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15617                started.display(),
15618                proceed.display()
15619            )),
15620            ..Default::default()
15621        }];
15622        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15623        let config = git::git(&repo, &["config", "--local", "--list"])
15624            .await
15625            .unwrap();
15626        let mut prep = Box::pin(runner.prep());
15627        assert!(
15628            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15629                .await
15630                .is_err()
15631        );
15632        assert!(
15633            !root.exists(),
15634            "contention must prevent even checkout creation"
15635        );
15636        assert!(!started.exists(), "contention must prevent setup");
15637        assert_eq!(
15638            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15639            branches
15640        );
15641        assert_eq!(
15642            git::git(&repo, &["config", "--local", "--list"])
15643                .await
15644                .unwrap(),
15645            config
15646        );
15647        drop(lock);
15648        tokio::select! {
15649            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15650            result = tokio::time::timeout(Duration::from_secs(10), async {
15651                while !started.exists() {
15652                    tokio::time::sleep(Duration::from_millis(20)).await;
15653                }
15654            }) => result.unwrap(),
15655        }
15656        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15657        assert_eq!(
15658            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15659            -1
15660        );
15661        assert!(
15662            tokio::time::timeout(
15663                Duration::from_millis(100),
15664                crate::repo_lock::SeatLease::for_checkout(
15665                    &repo,
15666                    true,
15667                    crate::config::test_coordination().as_ref()
15668                )
15669            )
15670            .await
15671            .is_err(),
15672            "preparation must also exclude local editors"
15673        );
15674        std::fs::write(proceed, "ready").unwrap();
15675        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15676            .await
15677            .unwrap()
15678            .unwrap();
15679        drop(prep);
15680        assert_eq!(runner.state.prep_complete, Some(true));
15681        assert!(root.join("judge-1").exists());
15682        assert!(root.join("advisor-1").exists());
15683        assert_eq!(
15684            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15685            0
15686        );
15687    }
15688
15689    #[tokio::test]
15690    #[cfg(unix)]
15691    async fn preparation_lock_failure_prevents_repository_changes() {
15692        ask_test_home();
15693        let tmp = tempfile::tempdir().unwrap();
15694        let repo = tmp.path().join("repo");
15695        std::fs::create_dir_all(&repo).unwrap();
15696        init_repo(&repo);
15697        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15698        // A directory at the required lock path makes opening the real lock
15699        // fail, rather than merely exercising an invalid registration.
15700        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15701        let mut runner = runner_at(RunStatus::Prep);
15702        runner.state.repo = repo.clone();
15703        runner.state.config.coordination = crate::config::test_coordination();
15704        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15705        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15706        runner.roles.implementers = vec![spec("implement")];
15707        let error = runner.prep().await.unwrap_err();
15708        assert!(
15709            error
15710                .to_string()
15711                .contains("acquire mandatory repository lock")
15712        );
15713        assert!(!runner.state.worktree_root().exists());
15714        assert!(runner.state.candidates.is_empty());
15715        assert!(!repo.join(".git/config.worktree").exists());
15716        assert_eq!(
15717            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15718            "* main"
15719        );
15720    }
15721
15722    #[tokio::test]
15723    async fn preparation_records_creation_intent_before_git_can_fail() {
15724        ask_test_home();
15725        let tmp = tempfile::tempdir().unwrap();
15726        let repo = tmp.path().join("repo");
15727        std::fs::create_dir_all(&repo).unwrap();
15728        init_repo(&repo);
15729        let mut runner = runner_at(RunStatus::Prep);
15730        runner.state.repo = repo.clone();
15731        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15732        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15733        runner.state.config.blind.commit_msg_hook = false;
15734        runner.roles.implementers = vec![spec("one")];
15735        let label = blind::assign_labels(1, runner.state.seed)[0];
15736        let branch = runner.state.branch_for(label);
15737        // A conflicting branch makes Git refuse creation before a candidate
15738        // can be registered. The durable intent must still identify it.
15739        git::git(&repo, &["branch", &branch]).await.unwrap();
15740        assert!(runner.prep().await.is_err());
15741        let saved = RunState::load(&runner.state.id).unwrap();
15742        assert!(saved.candidates.is_empty());
15743        let event = saved
15744            .events
15745            .iter()
15746            .find(|e| e.node == "prep-intent")
15747            .unwrap();
15748        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15749        assert_eq!(intent["branch"], branch);
15750        assert_eq!(intent["start"], runner.state.base_commit);
15751        assert_eq!(
15752            intent["worktree"],
15753            runner
15754                .state
15755                .worktree_root()
15756                .join(format!("cand-{label}"))
15757                .to_string_lossy()
15758                .as_ref()
15759        );
15760        assert_eq!(intent["picks"], serde_json::json!([]));
15761    }
15762
15763    #[tokio::test]
15764    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15765        ask_test_home();
15766        let tmp = tempfile::tempdir().unwrap();
15767        let repo = tmp.path().join("repo");
15768        std::fs::create_dir_all(&repo).unwrap();
15769        init_repo(&repo);
15770        let mut runner = runner_at(RunStatus::Prep);
15771        runner.state.repo = repo.clone();
15772        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15773        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15774        runner.state.sources.pick_commits.push(refs::Input {
15775            token: "missing".into(),
15776            sha: "0".repeat(40),
15777            provenance: refs::Provenance::ExplicitCli,
15778        });
15779        assert!(runner.prep().await.is_err());
15780        assert!(runner.state.candidates.is_empty());
15781        assert!(!runner.state.worktree_root().exists());
15782    }
15783
15784    #[tokio::test]
15785    async fn prose_and_legacy_inference_never_import_into_candidates() {
15786        ask_test_home();
15787        let tmp = tempfile::tempdir().unwrap();
15788        let repo = tmp.path().join("repo");
15789        std::fs::create_dir_all(&repo).unwrap();
15790        init_repo(&repo);
15791        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15792        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15793            .await
15794            .unwrap();
15795        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15796        git::git(&repo, &["add", "."]).await.unwrap();
15797        git::git(&repo, &["commit", "-m", "historical implementation"])
15798            .await
15799            .unwrap();
15800        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15801        git::git(&repo, &["checkout", "main"]).await.unwrap();
15802        for instruction in [
15803            format!("Example: magi/history/A at {source}"),
15804            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15805            format!("Never import magi/history/A or {source}"),
15806            format!("Conversation answer: I mentioned {source} only as background"),
15807        ] {
15808            let mut runner = runner_at(RunStatus::Prep);
15809            runner.state.repo = repo.clone();
15810            runner.state.base_commit = base.clone();
15811            runner.state.instruction = instruction.clone();
15812            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15813            runner.state.config.blind.commit_msg_hook = false;
15814            runner.state.seeds = vec![refs::Seed {
15815                token: "magi/history/A".into(),
15816                sha: source.clone(),
15817                kind: refs::SeedKind::Unmerged,
15818                branch: true,
15819                detail: String::new(),
15820            }];
15821            runner.roles.implementers = vec![spec("one")];
15822            assert_eq!(seeded_instruction(&runner.state), instruction);
15823            // Model an old record with inferred seeds but no structured input field.
15824            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15825            legacy.as_object_mut().unwrap().remove("sources");
15826            runner.state = serde_json::from_value(legacy).unwrap();
15827            assert_eq!(runner.state.sources, refs::Sources::default());
15828            runner.state.save().unwrap();
15829            runner.state = RunState::load(&runner.state.id).unwrap();
15830            runner.prep().await.unwrap();
15831            for candidate in &runner.state.candidates {
15832                assert_eq!(
15833                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15834                    base
15835                );
15836                assert!(!candidate.worktree.join("unwanted").exists());
15837            }
15838        }
15839    }
15840
15841    #[tokio::test]
15842    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15843        ask_test_home();
15844        let tmp = tempfile::tempdir().unwrap();
15845        let repo = tmp.path().join("repo");
15846        std::fs::create_dir_all(&repo).unwrap();
15847        init_repo(&repo);
15848        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15849        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15850        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15851        git::git(&repo, &["add", "."]).await.unwrap();
15852        git::git(&repo, &["commit", "-m", "required seed"])
15853            .await
15854            .unwrap();
15855        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15856        git::git(&repo, &["checkout", "main"]).await.unwrap();
15857        let mut runner = runner_at(RunStatus::Prep);
15858        runner.state.repo = repo.clone();
15859        runner.state.base_commit = base.clone();
15860        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15861        runner.state.config.graph.advise = false;
15862        runner.state.config.blind.commit_msg_hook = false;
15863        runner.state.sources =
15864            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15865                .await
15866                .unwrap();
15867        runner.roles.implementers = vec![spec("one"), spec("two")];
15868        runner.roles.judges = vec![spec("judge")];
15869        let label = blind::assign_labels(2, runner.state.seed)[0];
15870        let branch = runner.state.branch_for(label);
15871        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15872        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15873            .await
15874            .unwrap();
15875        runner.state.candidates.push(Candidate {
15876            index: 0,
15877            label,
15878            agent: "one".into(),
15879            branch,
15880            worktree,
15881            summary: String::new(),
15882            stat: String::new(),
15883            files: 0,
15884            commits: 0,
15885            empty: false,
15886            failed: None,
15887            verified_noop: None,
15888            duration_ms: 0,
15889            folded: false,
15890        });
15891        runner.state.save().unwrap();
15892        // Reload the checkpoint immediately before its first cherry-pick.
15893        runner.state = RunState::load(&runner.state.id).unwrap();
15894        assert_eq!(runner.state.prep_complete, Some(false));
15895        runner.prep().await.unwrap();
15896        assert_eq!(runner.state.prep_complete, Some(true));
15897        assert_eq!(runner.state.candidates.len(), 2);
15898        for candidate in &runner.state.candidates {
15899            assert_eq!(
15900                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15901                git::tree_of(&repo, &seed).await.unwrap()
15902            );
15903            assert_eq!(
15904                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15905                "required commit"
15906            );
15907        }
15908        assert!(runner.state.worktree_root().join("judge-1").exists());
15909        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15910        runner.prep().await.unwrap();
15911        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15912
15913        runner.state.prep_complete = Some(false);
15914        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15915            run: Some(format!(
15916                "test -f '{}'",
15917                tmp.path().join("dependency-ready").display()
15918            )),
15919            ..Default::default()
15920        }];
15921        assert!(runner.prep().await.is_err());
15922        assert!(runner.state.prep_failure.is_some());
15923        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15924        runner.state = RunState::load(&runner.state.id).unwrap();
15925        assert!(
15926            runner.prep().await.is_err(),
15927            "correcting the cause alone must not auto-retry"
15928        );
15929        let failed_at = runner.state.events.last().unwrap().at;
15930        assert!(
15931            runner.retry_preparation(failed_at).is_err(),
15932            "old approvals cannot unlock the hold"
15933        );
15934        let mut task = crate::queue::Task::new(
15935            "retry".into(),
15936            "retry setup".into(),
15937            repo.clone(),
15938            crate::queue::Source::Human,
15939        );
15940        task.start(runner.state.id.clone());
15941        task.hold_machine(Some("setup failed".into()));
15942        task.release_operator("cli", None);
15943        runner
15944            .retry_preparation(task.resumption.as_ref().unwrap().at)
15945            .unwrap();
15946        runner.prep().await.unwrap();
15947        assert!(runner.state.prep_failure.is_none());
15948        assert_eq!(runner.state.prep_complete, Some(true));
15949        assert_eq!(runner.state.candidates.len(), 2);
15950        assert!(runner.state.worktree_root().join("judge-1").exists());
15951        for candidate in &runner.state.candidates {
15952            assert_eq!(
15953                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15954                git::tree_of(&repo, &seed).await.unwrap()
15955            );
15956        }
15957    }
15958
15959    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15960        let mut heads = Vec::new();
15961        for candidate in candidates {
15962            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15963        }
15964        heads
15965    }
15966
15967    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15968        ask_test_home();
15969        let tmp = tempfile::tempdir().unwrap();
15970        let repo = tmp.path().join("repo");
15971        std::fs::create_dir(&repo).unwrap();
15972        init_repo(&repo);
15973        let mut runner = runner_at(RunStatus::Reviewing);
15974        runner.state.repo = repo.clone();
15975        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15976        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15977        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15978            let branch = format!("candidate-{label}");
15979            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15980            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15981                .await
15982                .unwrap();
15983            if label == 'B' {
15984                std::fs::write(worktree.join("change"), "winner").unwrap();
15985                git::rescue_commit(&worktree, "candidate change")
15986                    .await
15987                    .unwrap();
15988            }
15989            runner.state.candidates.push(Candidate {
15990                index,
15991                label,
15992                agent: "alpha".into(),
15993                branch,
15994                worktree,
15995                summary: "completed reply".into(),
15996                stat: String::new(),
15997                files: usize::from(label == 'B'),
15998                commits: usize::from(label == 'B'),
15999                empty: label == 'A',
16000                failed: None,
16001                verified_noop: None,
16002                duration_ms: 1,
16003                folded: false,
16004            });
16005        }
16006        runner.state.judge_skipped = true;
16007        runner.state.tally = Some(Tally {
16008            first_choice: BTreeMap::new(),
16009            borda: BTreeMap::new(),
16010            winner: 'B',
16011            rankings: 0,
16012            unanimous_initial: false,
16013            deliberated: false,
16014            changed_votes: 0,
16015            unanimous_final: false,
16016            tie_break: None,
16017            judges: 0,
16018            present: 0,
16019            quorum: 0,
16020            met_quorum: true,
16021            uncontested: Some("only B produced a change".into()),
16022        });
16023        runner.state.save().unwrap();
16024        (tmp, runner)
16025    }
16026
16027    #[tokio::test]
16028    async fn fold_losers_persists_discard_before_later_teardown_failure() {
16029        let (_tmp, mut runner) = preservation_resume_fixture().await;
16030        runner.state.candidates[0].folded = true;
16031        runner.state.config.graph.advise = true;
16032        runner.state.config.graph.advisors = 2;
16033        let repo = runner.state.repo.clone();
16034        let root = runner.state.worktree_root();
16035        let discarded = root.join("advisor-1");
16036        git::worktree_add_detached(&repo, &discarded, "main")
16037            .await
16038            .unwrap();
16039        std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
16040        let protected = root.join("advisor-2");
16041        git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
16042            .await
16043            .unwrap();
16044        std::fs::write(protected.join("keep"), "uncommitted").unwrap();
16045        runner.state.save().unwrap();
16046
16047        assert!(runner.fold_losers().await.is_err());
16048        assert!(!discarded.exists());
16049        assert!(protected.join("keep").exists());
16050        let saved = RunState::load(&runner.state.id).unwrap();
16051        assert!(saved.events.iter().any(|event| {
16052            event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
16053        }));
16054    }
16055
16056    #[tokio::test]
16057    async fn read_seat_teardown_discards_only_owned_detached_seats() {
16058        let (tmp, mut runner) = preservation_resume_fixture().await;
16059        let repo = runner.state.repo.clone();
16060        let root = runner.state.worktree_root();
16061        for name in ["advisor-3", "judge-1", "review-2"] {
16062            let path = root.join(name);
16063            git::worktree_add_detached(&repo, &path, "main")
16064                .await
16065                .unwrap();
16066            std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
16067            std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
16068            remove_read_seat_worktree(&mut runner.state, &path)
16069                .await
16070                .unwrap();
16071            assert!(!path.exists());
16072        }
16073        let outside = tmp.path().join("advisor-1");
16074        let unknown = root.join("unknown");
16075        let attached = root.join("advisor-4");
16076        git::worktree_add_detached(&repo, &outside, "main")
16077            .await
16078            .unwrap();
16079        git::worktree_add_detached(&repo, &unknown, "main")
16080            .await
16081            .unwrap();
16082        git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16083            .await
16084            .unwrap();
16085        let foreign = tmp.path().join("foreign");
16086        std::fs::create_dir(&foreign).unwrap();
16087        init_repo(&foreign);
16088        let wrong_repo = root.join("judge-2");
16089        git::worktree_add_detached(&foreign, &wrong_repo, "main")
16090            .await
16091            .unwrap();
16092        let candidate = runner.state.candidates[0].worktree.clone();
16093        for path in [outside, unknown, attached, wrong_repo, candidate] {
16094            std::fs::write(path.join("uncommitted"), "keep").unwrap();
16095            assert!(
16096                remove_read_seat_worktree(&mut runner.state, &path)
16097                    .await
16098                    .is_err()
16099            );
16100            assert_eq!(
16101                std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16102                "keep"
16103            );
16104        }
16105    }
16106
16107    #[tokio::test]
16108    #[cfg(unix)]
16109    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16110        for (solo, failed_checkpoint) in
16111            [(true, false), (false, false), (true, true), (false, true)]
16112        {
16113            let (_tmp, mut runner) = preservation_resume_fixture().await;
16114            if solo {
16115                runner.state.candidates.retain(|c| c.label == 'A');
16116            }
16117            runner.state.candidates[0].empty = false;
16118            runner.state.candidates[0].summary.clear();
16119            runner.state.tally = None;
16120            runner.state.judge_skipped = false;
16121            runner.state.status = RunStatus::Implementing;
16122            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16123            let mut implementer = spec("alpha");
16124            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16125            runner.roles.implementers = vec![implementer];
16126            let patch = runner.state.dir().join("artifacts/cand-A.patch");
16127            if failed_checkpoint {
16128                std::fs::create_dir_all(&patch).unwrap();
16129            }
16130            runner.state.save().unwrap();
16131            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16132            drop(pressure);
16133            runner.state = RunState::load(&runner.state.id).unwrap();
16134            assert_eq!(runner.state.status, RunStatus::Blocked);
16135            assert!(runner.state.interrupted_implementations.contains(&'A'));
16136            assert_eq!(runner.state.candidates[0].commits, 0);
16137            assert!(!runner.state.candidates[0].empty);
16138            assert!(runner.state.candidates[0].failed.is_none());
16139            if failed_checkpoint {
16140                assert!(runner.state.candidate_preservation.contains_key(&'A'));
16141                std::fs::remove_dir(patch).unwrap();
16142            }
16143            runner.state.resource_block = None;
16144            runner.state.config.disk.min_free_bytes = 0;
16145            runner.roles.implementers[0].command = vec![
16146                "sh".into(),
16147                "-c".into(),
16148                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16149            ];
16150            runner.implement().await.unwrap();
16151            assert!(runner.state.interrupted_implementations.is_empty());
16152            assert!(runner.state.candidate_preservation.is_empty());
16153            assert_eq!(runner.state.candidates[0].commits, 1);
16154            assert!(!runner.state.candidates[0].empty);
16155            assert!(
16156                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16157                    .unwrap()
16158                    .contains("retried empty interrupted seat")
16159            );
16160            if !solo {
16161                assert_eq!(runner.state.candidates[1].commits, 1);
16162            }
16163        }
16164    }
16165
16166    #[tokio::test]
16167    #[cfg(unix)]
16168    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16169        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16170            let (_tmp, mut runner) = preservation_resume_fixture().await;
16171            runner.state.candidates.retain(|c| c.label == 'A');
16172            runner.state.candidates[0].empty = false;
16173            runner.state.candidates[0].summary.clear();
16174            runner.state.tally = None;
16175            runner.state.judge_skipped = false;
16176            runner.state.status = RunStatus::Implementing;
16177            // Force only the real post-spawn watchdog, without failing the
16178            // separate admission check. Existing edits predate its measurement.
16179            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16180            let wt = runner.state.candidates[0].worktree.clone();
16181            std::fs::write(
16182                wt.join("interrupted-source"),
16183                "retain implementation work\n",
16184            )
16185            .unwrap();
16186            if committed {
16187                git::rescue_commit(&wt, "implementation checkpoint")
16188                    .await
16189                    .unwrap();
16190            }
16191            let lock = git::git(
16192                &wt,
16193                &[
16194                    "rev-parse",
16195                    "--path-format=absolute",
16196                    "--git-path",
16197                    "index.lock",
16198                ],
16199            )
16200            .await
16201            .unwrap();
16202            if block_rescue {
16203                std::fs::write(&lock, "fixture-owned lock").unwrap();
16204            }
16205            let mut implementer = spec("alpha");
16206            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16207            runner.roles.implementers = vec![implementer];
16208            runner.state.save().unwrap();
16209            let result = runner.implement().await;
16210            drop(pressure);
16211            assert_eq!(result.is_err(), block_rescue);
16212            assert_eq!(
16213                runner.state.resource_block.as_ref().unwrap().kind,
16214                "disk pressure"
16215            );
16216            assert!(
16217                runner
16218                    .state
16219                    .events
16220                    .iter()
16221                    .any(|e| e.message.contains("agent execution interrupted"))
16222            );
16223            assert!(
16224                runner
16225                    .state
16226                    .dir()
16227                    .join("artifacts/candidate-A-recovery/staged.patch")
16228                    .exists()
16229            );
16230            if block_rescue {
16231                let saved = RunState::load(&runner.state.id).unwrap();
16232                assert!(saved.candidate_preservation.contains_key(&'A'));
16233                assert!(!saved.candidates[0].empty);
16234                assert_eq!(
16235                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16236                    "retain implementation work\n"
16237                );
16238                std::fs::remove_file(lock).unwrap();
16239                runner.state = saved;
16240            } else {
16241                assert_eq!(runner.state.status, RunStatus::Blocked);
16242                assert_eq!(runner.state.candidates[0].commits, 1);
16243            }
16244            // Recovery must save the candidate before deciding whether another
16245            // implementation is needed. No implementer is available on retry.
16246            runner.roles.implementers.clear();
16247            runner.state.config.disk.min_free_bytes = 0;
16248            runner.state.resource_block = None;
16249            runner.implement().await.unwrap();
16250            assert!(runner.state.candidate_preservation.is_empty());
16251            assert_eq!(runner.state.candidates[0].commits, 1);
16252            assert!(!runner.state.candidates[0].empty);
16253            assert!(git::is_clean(&wt).await.unwrap());
16254            assert!(
16255                crate::run::read_artifact(&runner.state, "cand-A.patch")
16256                    .unwrap()
16257                    .contains("retain implementation work")
16258            );
16259        }
16260    }
16261
16262    #[tokio::test]
16263    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16264        let (_tmp, mut runner) = preservation_resume_fixture().await;
16265        // Schema 15 already confirmed the unfolded no-change result.
16266        runner.implement().await.unwrap();
16267        assert!(runner.state.candidate_preservation.is_empty());
16268        assert!(
16269            !runner
16270                .state
16271                .dir()
16272                .join("artifacts/candidate-A-recovery")
16273                .exists()
16274        );
16275        runner.fold_losers().await.unwrap();
16276        assert!(!runner.state.candidates[0].worktree.exists());
16277        for schema in [14, crate::run::SCHEMA] {
16278            runner.state.schema = schema;
16279            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16280            runner.state.save().unwrap();
16281            runner.state = RunState::load(&runner.state.id).unwrap();
16282            runner.implement().await.unwrap();
16283            assert!(runner.state.candidate_preservation.is_empty());
16284            assert!(runner.state.candidates[0].folded);
16285            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16286        }
16287    }
16288
16289    #[tokio::test]
16290    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16291        let (_tmp, mut runner) = preservation_resume_fixture().await;
16292        runner.state.candidates.retain(|c| c.label == 'A');
16293        runner.state.tally = None;
16294        runner.state.judge_skipped = false;
16295        let candidate = &mut runner.state.candidates[0];
16296        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16297        candidate.failed = Some("deployment validation unavailable".into());
16298        std::fs::write(
16299            candidate.worktree.join("repair"),
16300            "future launcher policy\n",
16301        )
16302        .unwrap();
16303        runner
16304            .state
16305            .candidate_preservation
16306            .insert('A', "awaiting persistence".into());
16307        runner.persist_candidates().await.unwrap();
16308        let candidate = &runner.state.candidates[0];
16309        assert!(!candidate.empty);
16310        assert_eq!(candidate.commits, 1);
16311        assert_eq!(candidate.files, 1);
16312        assert!(candidate.failed.is_none());
16313        assert!(candidate.summary.contains("EPERM"));
16314        assert!(
16315            crate::run::read_artifact(&runner.state, "cand-A.patch")
16316                .unwrap()
16317                .contains("future launcher policy")
16318        );
16319        assert!(runner.state.candidate_preservation.is_empty());
16320    }
16321
16322    #[tokio::test]
16323    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16324        let (_tmp, mut runner) = preservation_resume_fixture().await;
16325        let wt = runner.state.candidates[0].worktree.clone();
16326        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16327        runner.state.schema = 14;
16328        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16329        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16330        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16331        for _ in 0..2 {
16332            assert!(runner.implement().await.is_err());
16333            assert_eq!(runner.state.status, RunStatus::Blocked);
16334            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16335            assert!(
16336                runner.state.candidates[0].empty,
16337                "old verdict must remain unusable"
16338            );
16339            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16340            assert!(runner.state.judge_skipped);
16341            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16342            assert!(
16343                crate::run::read_artifact(&runner.state, "cand-A.patch")
16344                    .unwrap()
16345                    .contains("owned edits")
16346            );
16347            assert!(wt.exists());
16348            assert_eq!(
16349                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16350                    .await
16351                    .unwrap(),
16352                1
16353            );
16354            runner.state = RunState::load(&runner.state.id).unwrap();
16355        }
16356    }
16357
16358    #[tokio::test]
16359    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16360        let (_tmp, mut runner) = preservation_resume_fixture().await;
16361        let wt = runner.state.candidates[0].worktree.clone();
16362        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16363        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16364        let index_lock = PathBuf::from(
16365            git::git(
16366                &wt,
16367                &[
16368                    "rev-parse",
16369                    "--path-format=absolute",
16370                    "--git-path",
16371                    "index.lock",
16372                ],
16373            )
16374            .await
16375            .unwrap(),
16376        );
16377        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16378        runner.state.schema = 14;
16379        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16380        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16381        assert!(runner.implement().await.is_err());
16382        assert!(runner.after_implement().is_err());
16383        assert!(runner.fold_losers().await.is_err());
16384        assert_eq!(
16385            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16386            "failed seat edits"
16387        );
16388        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16389        runner.state = RunState::load(&runner.state.id).unwrap();
16390        std::fs::remove_file(index_lock).unwrap();
16391        // Recovery finds changes that the old uncontested result never judged.
16392        assert!(runner.implement().await.is_err());
16393        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16394        assert!(runner.state.candidates[0].empty);
16395        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16396        assert!(
16397            crate::run::read_artifact(&runner.state, "cand-A.patch")
16398                .unwrap()
16399                .contains("failed seat edits")
16400        );
16401        assert!(wt.exists());
16402    }
16403
16404    #[tokio::test]
16405    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16406        let (_tmp, mut runner) = preservation_resume_fixture().await;
16407        let repo = runner.state.repo.clone();
16408        let common = git::git(
16409            &repo,
16410            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16411        )
16412        .await
16413        .unwrap();
16414        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16415        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16416        assert!(runner.fold_losers().await.is_err());
16417        assert!(!runner.state.candidates[0].worktree.exists());
16418        assert!(!runner.state.candidates[0].folded);
16419        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16420        runner.state = RunState::load(&runner.state.id).unwrap();
16421        std::fs::remove_file(branch_lock).unwrap();
16422        let judge = runner.state.worktree_root().join("judge-1");
16423        git::worktree_add_detached(&repo, &judge, "main")
16424            .await
16425            .unwrap();
16426        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16427        git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16428            .await
16429            .unwrap();
16430        runner.roles.judges = vec![spec("judge")];
16431        assert!(runner.fold_losers().await.is_err());
16432        runner.state = RunState::load(&runner.state.id).unwrap();
16433        assert!(
16434            runner.state.candidates[0].folded,
16435            "candidate progress saved before judge failure"
16436        );
16437        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16438        assert!(judge.join("uncommitted").exists());
16439        git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16440            .await
16441            .unwrap();
16442        runner.fold_losers().await.unwrap();
16443        assert!(!judge.exists());
16444        runner.fold_losers().await.unwrap();
16445    }
16446
16447    #[tokio::test]
16448    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16449        let (_tmp, mut runner) = preservation_resume_fixture().await;
16450        runner.state.candidates.retain(|c| c.label == 'B');
16451        runner.state.tally = None;
16452        runner.state.judge_skipped = false;
16453        runner.state.prep_complete = Some(true);
16454        runner.state.config.graph.advise = false;
16455        runner.state.config.graph.selected_implementer = Some("alpha".into());
16456        let wt = runner.state.candidates[0].worktree.clone();
16457        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16458        runner
16459            .state
16460            .candidate_preservation
16461            .insert('B', "awaiting persistence".into());
16462        let index_lock = git::git(
16463            &wt,
16464            &[
16465                "rev-parse",
16466                "--path-format=absolute",
16467                "--git-path",
16468                "index.lock",
16469            ],
16470        )
16471        .await
16472        .unwrap();
16473        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16474        runner.state.save().unwrap();
16475        assert!(runner.execute_graph().await.is_err());
16476        assert_eq!(runner.state.status, RunStatus::Blocked);
16477        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16478        std::fs::remove_file(index_lock).unwrap();
16479        // Park at the next graph boundary: this tests the real selected-seat
16480        // hold guard without dispatching reviewers or publishing anything.
16481        for _ in 0..2 {
16482            runner.state = RunState::load(&runner.state.id).unwrap();
16483            runner.pause.park();
16484            runner.execute_graph().await.unwrap();
16485            assert_eq!(runner.state.status, RunStatus::Implementing);
16486            assert!(runner.state.candidate_preservation.is_empty());
16487            assert_eq!(runner.state.candidates[0].commits, 2);
16488            assert!(runner.state.tally.is_none());
16489        }
16490        runner.implement().await.unwrap();
16491        assert_eq!(runner.state.status, RunStatus::Judging);
16492        // A genuine selected-implementer hold must still require release.
16493        runner.state.status = RunStatus::Blocked;
16494        runner.state.candidates[0].failed = Some("implementation failed".into());
16495        runner.state.save().unwrap();
16496        let error = runner.execute_graph().await.unwrap_err();
16497        assert!(
16498            error
16499                .to_string()
16500                .contains("selected implementation is held")
16501        );
16502    }
16503
16504    #[tokio::test]
16505    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16506        let (_tmp, mut runner) = preservation_resume_fixture().await;
16507        let repo = runner.state.repo.clone();
16508        let candidate = runner.state.candidates[0].clone();
16509        // Remove only the disposable fixture directory, leaving Git's linked
16510        // worktree registration and branch intact, as in a vanished checkout.
16511        let registered = git::worktree_holding(&repo, &candidate.branch)
16512            .await
16513            .unwrap()
16514            .unwrap();
16515        assert_eq!(
16516            registered.canonicalize().unwrap(),
16517            candidate.worktree.canonicalize().unwrap()
16518        );
16519        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16520        assert_eq!(
16521            git::worktree_holding(&repo, &candidate.branch)
16522                .await
16523                .unwrap(),
16524            Some(registered)
16525        );
16526        assert!(
16527            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16528            "the stale registration initially keeps the branch checked out"
16529        );
16530        runner.fold_losers().await.unwrap();
16531        assert!(runner.state.candidates[0].folded);
16532        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16533        assert_eq!(
16534            git::worktree_holding(&repo, &candidate.branch)
16535                .await
16536                .unwrap(),
16537            None
16538        );
16539        runner.state = RunState::load(&runner.state.id).unwrap();
16540        runner.fold_losers().await.unwrap();
16541    }
16542
16543    #[tokio::test]
16544    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16545        let (_tmp, mut runner) = preservation_resume_fixture().await;
16546        let repo = runner.state.repo.clone();
16547        let candidate = runner.state.candidates[0].clone();
16548        let winner = runner.state.candidates[1].clone();
16549        let registered = git::worktree_holding(&repo, &candidate.branch)
16550            .await
16551            .unwrap()
16552            .unwrap();
16553        assert_eq!(
16554            registered.canonicalize().unwrap(),
16555            candidate.worktree.canonicalize().unwrap()
16556        );
16557        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16558        assert_eq!(
16559            git::worktree_holding(&repo, &candidate.branch)
16560                .await
16561                .unwrap(),
16562            Some(registered)
16563        );
16564        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16565
16566        for _ in 0..2 {
16567            fold_run(&mut runner.state, false, &crate::run::home())
16568                .await
16569                .unwrap();
16570            runner.state = RunState::load(&runner.state.id).unwrap();
16571            assert!(runner.state.candidates[0].folded);
16572            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16573            assert_eq!(
16574                git::worktree_holding(&repo, &candidate.branch)
16575                    .await
16576                    .unwrap(),
16577                None
16578            );
16579            assert!(!runner.state.candidates[1].folded);
16580            assert!(winner.worktree.exists());
16581            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16582        }
16583    }
16584    #[tokio::test]
16585    async fn fold_recovery_override_archives_sources_and_keeps_safety_checks() {
16586        for sidecar in [false, true] {
16587            let tmp = tempfile::tempdir().unwrap();
16588            let home = tmp.path().join("home");
16589            let repo = tmp.path().join("repo");
16590            std::fs::create_dir(&repo).unwrap();
16591            init_repo(&repo);
16592            let mut state = runner_at(RunStatus::Superseded).state;
16593            state.repo = repo.clone();
16594            state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16595            state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16596            state.driver_exited = true;
16597            state.reserve_under(&home).unwrap();
16598            git::git(
16599                &repo,
16600                &["update-ref", "refs/remotes/origin/main", &state.base_commit],
16601            )
16602            .await
16603            .unwrap();
16604            let wt = state.worktree_root().join("cand-A");
16605            git::worktree_add_branch(&repo, &wt, "candidate", "main")
16606                .await
16607                .unwrap();
16608            std::fs::write(wt.join("independent"), "retained implementation\n").unwrap();
16609            git::commit_all(&wt, "fixture: independent source")
16610                .await
16611                .unwrap();
16612            let head = git::rev_parse(&wt, "HEAD").await.unwrap();
16613            state.candidates.push(Candidate {
16614                index: 0,
16615                label: 'A',
16616                agent: "fixture".into(),
16617                branch: "candidate".into(),
16618                worktree: wt.clone(),
16619                summary: String::new(),
16620                stat: String::new(),
16621                files: 0,
16622                commits: 0,
16623                empty: false,
16624                failed: None,
16625                verified_noop: None,
16626                duration_ms: 0,
16627                folded: false,
16628            });
16629            let mut task = crate::queue::Task::new(
16630                "task".into(),
16631                "task".into(),
16632                repo.clone(),
16633                crate::queue::Source::Human,
16634            );
16635            task.runs.push(state.id.clone());
16636            crate::queue::Queue::at(home.join("queue"))
16637                .put(&mut task)
16638                .unwrap();
16639            if sidecar {
16640                crate::pr_recovery::retain_sources(&task, &home).unwrap();
16641            }
16642            state.save_under(&home).unwrap();
16643            let reason = crate::pr_recovery::source_retention_reason(&home, &state).unwrap();
16644            assert!(
16645                fold_run(&mut state, true, &home)
16646                    .await
16647                    .unwrap_err()
16648                    .to_string()
16649                    .contains(&reason)
16650            );
16651            let mut disk = state.config.disk.clone();
16652            disk.fold_grace_secs = 0;
16653            assert_eq!(
16654                crate::clean::fold_due(
16655                    &home.join("runs"),
16656                    &home,
16657                    &state.worktree_root(),
16658                    &disk,
16659                    Timestamp::now() + std::time::Duration::from_secs(10),
16660                )
16661                .await
16662                .unwrap(),
16663                (0, 0)
16664            );
16665            assert!(wt.exists());
16666            state.status = RunStatus::Blocked;
16667            state.save_under(&home).unwrap();
16668            assert!(
16669                fold_run_overriding_recovery_retention(&mut state, true, &home)
16670                    .await
16671                    .is_err()
16672            );
16673            state.status = RunStatus::Superseded;
16674            state.driver_exited = false;
16675            state.driver_claimed = None;
16676            state.save_under(&home).unwrap();
16677            assert!(
16678                fold_run_overriding_recovery_retention(&mut state, true, &home)
16679                    .await
16680                    .is_err()
16681            );
16682            state.driver_pid = Some(std::process::id());
16683            state.driver_started_at = crate::proc::process_started_at(std::process::id());
16684            state.save_under(&home).unwrap();
16685            assert!(
16686                fold_run_overriding_recovery_retention(&mut state, true, &home)
16687                    .await
16688                    .is_err()
16689            );
16690            state.driver_exited = true;
16691            state.driver_pid = None;
16692            state.driver_started_at = None;
16693            state.save_under(&home).unwrap();
16694            let lease = crate::driver::maintenance(&home, &state.id).unwrap();
16695            assert!(
16696                fold_run_overriding_recovery_retention(&mut state, true, &home)
16697                    .await
16698                    .is_err()
16699            );
16700            drop(lease);
16701            #[cfg(unix)]
16702            {
16703                let output = crate::retention::OutputLease::take(&wt, false).unwrap();
16704                assert!(
16705                    fold_run_overriding_recovery_retention(&mut state, true, &home)
16706                        .await
16707                        .is_err()
16708                );
16709                drop(output);
16710            }
16711            let owner = crate::pr_head::owner_lock(&home, &state.id).unwrap();
16712            assert!(
16713                fold_run_overriding_recovery_retention(&mut state, true, &home)
16714                    .await
16715                    .is_err()
16716            );
16717            drop(owner);
16718            let checkpoint = git::git(
16719                &wt,
16720                &[
16721                    "rev-parse",
16722                    "--path-format=absolute",
16723                    "--git-path",
16724                    "CHERRY_PICK_HEAD",
16725                ],
16726            )
16727            .await
16728            .unwrap();
16729            std::fs::write(&checkpoint, &head).unwrap();
16730            assert!(
16731                fold_run_overriding_recovery_retention(&mut state, true, &home)
16732                    .await
16733                    .is_err()
16734            );
16735            std::fs::remove_file(checkpoint).unwrap();
16736            std::fs::write(wt.join("dirty"), "keep").unwrap();
16737            assert!(
16738                fold_run_overriding_recovery_retention(&mut state, true, &home)
16739                    .await
16740                    .is_err()
16741            );
16742            assert!(wt.join("dirty").exists());
16743            std::fs::remove_file(wt.join("dirty")).unwrap();
16744            let evidence = home
16745                .join("runs")
16746                .join(&state.id)
16747                .join("artifacts/fold-evidence");
16748            std::fs::create_dir_all(evidence.parent().unwrap()).unwrap();
16749            std::fs::write(&evidence, "blocked archive").unwrap();
16750            assert!(
16751                fold_run_overriding_recovery_retention(&mut state, true, &home)
16752                    .await
16753                    .is_err()
16754            );
16755            assert!(wt.exists());
16756            assert!(
16757                !RunState::load_under(&state.id, &home)
16758                    .unwrap()
16759                    .events
16760                    .iter()
16761                    .any(|event| event.message.contains("--override-recovery-retention"))
16762            );
16763            std::fs::remove_file(&evidence).unwrap();
16764            fold_run_overriding_recovery_retention(&mut state, true, &home)
16765                .await
16766                .unwrap();
16767            assert!(!wt.exists());
16768            assert!(!git::branch_exists(&repo, "candidate").await.unwrap());
16769            assert!(
16770                git::worktree_holding(&repo, "candidate")
16771                    .await
16772                    .unwrap()
16773                    .is_none()
16774            );
16775            assert!(
16776                std::fs::read_to_string(evidence.join(format!("{head}.patch")))
16777                    .unwrap()
16778                    .contains("retained implementation")
16779            );
16780            assert!(evidence.join(format!("{head}.index")).is_file());
16781            let saved = RunState::load_under(&state.id, &home).unwrap();
16782            assert_eq!(saved.status, RunStatus::Superseded);
16783            assert!(saved.candidates[0].folded);
16784            assert!(saved.events.iter().any(|event| {
16785                event.message.contains("--override-recovery-retention")
16786                    && event.message.contains(&state.id)
16787                    && event.message.contains(&reason)
16788            }));
16789            assert_eq!(
16790                crate::pr_recovery::explicit_sources_retained(&home, &state.id),
16791                sidecar
16792            );
16793        }
16794    }
16795
16796    #[tokio::test]
16797    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16798        ask_test_home();
16799        let tmp = tempfile::tempdir().unwrap();
16800        let repo = tmp.path().join("repo");
16801        std::fs::create_dir(&repo).unwrap();
16802        init_repo(&repo);
16803        let mut runner = runner_at(RunStatus::Implementing);
16804        runner.state.repo = repo.clone();
16805        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16806        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16807        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16808            let branch = format!("candidate-{label}");
16809            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16810            git::worktree_add_branch(&repo, &wt, &branch, "main")
16811                .await
16812                .unwrap();
16813            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16814            git::git(&wt, &["add", "README.md"]).await.unwrap();
16815            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16816            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16817            runner.state.candidates.push(Candidate {
16818                index,
16819                label,
16820                agent: "alpha".to_owned(),
16821                branch,
16822                worktree: wt,
16823                summary: "completed implementation".to_owned(),
16824                stat: String::new(),
16825                files: 0,
16826                commits: 0,
16827                empty: false,
16828                failed: None,
16829                verified_noop: None,
16830                duration_ms: 42,
16831                folded: false,
16832            });
16833            runner
16834                .state
16835                .candidate_preservation
16836                .insert(label, "awaiting persistence".to_owned());
16837        }
16838        let wt = runner.state.candidates[0].worktree.clone();
16839        let index_lock = git::git(
16840            &wt,
16841            &[
16842                "rev-parse",
16843                "--path-format=absolute",
16844                "--git-path",
16845                "index.lock",
16846            ],
16847        )
16848        .await
16849        .unwrap();
16850        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16851        runner.state.save().unwrap();
16852        assert!(runner.persist_candidates().await.is_err());
16853        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16854        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16855        assert_eq!(
16856            runner.state.candidates[1].commits, 1,
16857            "other results are still collected"
16858        );
16859        assert!(
16860            !runner.state.candidates[0].empty,
16861            "failed rescue is not no change"
16862        );
16863        assert_eq!(
16864            runner.state.candidates[0].summary,
16865            "completed implementation"
16866        );
16867        assert!(runner.after_implement().is_err());
16868        let home = tmp.path().join("fold-home");
16869        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16870        assert!(wt.join("new-file").exists());
16871        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16872        assert!(recovery.join("staged.patch").exists());
16873        assert!(recovery.join("unstaged.patch").exists());
16874        assert_eq!(
16875            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16876            "untracked edits\n"
16877        );
16878        runner.state.begin_pass();
16879        runner.state.end_pass_parked(ParkCause::Upgrade {
16880            from: "1.0".into(),
16881            to: Some("1.1".into()),
16882        });
16883        runner.state.save().unwrap();
16884        runner.state = RunState::load(&runner.state.id).unwrap();
16885        assert!(matches!(
16886            runner.state.passes.last().unwrap().end,
16887            Some(crate::run::PassEnd::Parked {
16888                cause: ParkCause::Upgrade { .. }
16889            })
16890        ));
16891        std::fs::remove_file(index_lock).unwrap();
16892        // No implementer is configured: a persistence retry cannot dispatch a seat.
16893        runner.implement().await.unwrap();
16894        assert!(runner.state.candidate_preservation.is_empty());
16895        assert!(
16896            runner
16897                .state
16898                .candidates
16899                .iter()
16900                .all(|c| c.commits == 1 && !c.empty)
16901        );
16902        assert!(git::is_clean(&wt).await.unwrap());
16903        assert!(
16904            crate::run::read_artifact(&runner.state, "cand-A.patch")
16905                .unwrap()
16906                .contains("later tracked edits")
16907        );
16908    }
16909
16910    #[tokio::test]
16911    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16912        ask_test_home();
16913        let tmp = tempfile::tempdir().unwrap();
16914        let repo = tmp.path().join("repo");
16915        std::fs::create_dir(&repo).unwrap();
16916        init_repo(&repo);
16917        let wt = tmp.path().join("candidate");
16918        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16919            .await
16920            .unwrap();
16921        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16922        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16923        assert!(wt.join("uncommitted").exists());
16924        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16925        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16926        let mut runner = runner_at(RunStatus::Implementing);
16927        runner.state.repo = repo.clone();
16928        runner.state.base_commit = "missing-base".to_owned();
16929        runner.state.candidates.push(Candidate {
16930            index: 0,
16931            label: 'A',
16932            agent: "alpha".into(),
16933            branch: "candidate".into(),
16934            worktree: wt.clone(),
16935            summary: "completed".into(),
16936            stat: String::new(),
16937            files: 0,
16938            commits: 0,
16939            empty: false,
16940            failed: None,
16941            verified_noop: None,
16942            duration_ms: 1,
16943            folded: false,
16944        });
16945        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16946        assert!(
16947            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16948                .await
16949                .is_err()
16950        );
16951        assert!(!runner.state.candidates[0].folded);
16952        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16953        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16954        runner
16955            .state
16956            .candidate_preservation
16957            .insert('A', "pending".into());
16958        assert!(runner.persist_candidates().await.is_err());
16959        assert!(!runner.state.candidates[0].empty);
16960        assert!(!runner.state.candidates[0].folded);
16961        assert!(wt.exists());
16962        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16963        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16964        std::fs::create_dir_all(&patch_path).unwrap();
16965        assert!(
16966            runner.persist_candidates().await.is_err(),
16967            "artifact failure stays pending"
16968        );
16969        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16970        assert!(!runner.state.candidates[0].empty);
16971        std::fs::remove_dir(patch_path).unwrap();
16972        runner.persist_candidates().await.unwrap();
16973        assert!(
16974            runner.state.candidates[0].empty,
16975            "only successful inspection confirms no change"
16976        );
16977    }
16978
16979    fn init_repo(dir: &Path) {
16980        let run = |args: &[&str]| {
16981            let out = std::process::Command::new("git")
16982                .args(args)
16983                .current_dir(dir)
16984                .quiet()
16985                .output()
16986                .expect("spawn git");
16987            assert!(
16988                out.status.success(),
16989                "git {args:?} failed: {}",
16990                String::from_utf8_lossy(&out.stderr)
16991            );
16992        };
16993        run(&["init", "-b", "main"]);
16994        run(&["config", "user.name", "magi test"]);
16995        run(&["config", "user.email", "magi@example.com"]);
16996        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16997        run(&["add", "-A"]);
16998        run(&["commit", "-m", "init"]);
16999    }
17000
17001    // `settle_questions` is what closes the ghost the phone showed: a run's
17002    // seat asked something, the run then ended, and nothing was left to
17003    // abandon the question it left `open`. `HOME` is a process-wide
17004    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
17005    // first time it runs in the binary — every test below still reaches the
17006    // same directory whichever call won, and each gets its own run id from
17007    // `RunState::new`, so they never collide there.
17008    fn ask_test_home() {
17009        crate::run::pin_test_home();
17010    }
17011
17012    #[test]
17013    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
17014        ask_test_home();
17015        let mut runner = runner_at(RunStatus::Prep);
17016        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17017        // A blocked temporary output path makes the terminal save fail while
17018        // preserving the already published reservation.
17019        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
17020        let error = guard
17021            .finish(
17022                &mut runner.state,
17023                Err(
17024                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
17025                        .into(),
17026                ),
17027            )
17028            .unwrap_err();
17029        assert_eq!(
17030            error.downcast_ref::<std::io::Error>().unwrap().kind(),
17031            std::io::ErrorKind::PermissionDenied
17032        );
17033        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
17034        let message = format!("{error:#}");
17035        assert!(message.contains("original refusal"));
17036        assert!(!message.contains("could not save terminal"));
17037        assert_eq!(
17038            error.chain().count(),
17039            2,
17040            "only startup attribution wraps the original error"
17041        );
17042        let saved = RunState::load(&runner.state.id).unwrap();
17043        assert_eq!(saved.status, RunStatus::Blocked);
17044        assert!(!saved.startup.unwrap().complete);
17045    }
17046
17047    #[test]
17048    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
17049        ask_test_home();
17050        let mut runner = runner_at(RunStatus::Prep);
17051        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17052        let id = runner.state.id.clone();
17053        drop(guard);
17054        let saved = RunState::load(&id).unwrap();
17055        assert_eq!(saved.status, RunStatus::Blocked);
17056        assert!(!saved.startup.as_ref().unwrap().complete);
17057        assert!(
17058            saved
17059                .startup
17060                .unwrap()
17061                .failure
17062                .unwrap()
17063                .contains("interrupted")
17064        );
17065        assert!(Runner::resume(&id).is_err());
17066    }
17067
17068    #[test]
17069    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
17070        ask_test_home();
17071        let mut runner = runner_at(RunStatus::Prep);
17072        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17073        let error = guard
17074            .finish(
17075                &mut runner.state,
17076                Err(
17077                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
17078                        .into(),
17079                ),
17080            )
17081            .unwrap_err();
17082        assert_eq!(
17083            error.downcast_ref::<std::io::Error>().unwrap().kind(),
17084            std::io::ErrorKind::PermissionDenied
17085        );
17086        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
17087        let mut task = crate::queue::Task::new(
17088            "review".into(),
17089            "criteria".into(),
17090            runner.state.repo.clone(),
17091            crate::queue::Source::Human,
17092        );
17093        task.record_startup_failure(failure.clone());
17094        task.record_startup_failure(failure.clone());
17095        assert_eq!(task.startup_failures.len(), 1);
17096        assert!(task.runs.is_empty());
17097        assert_eq!(task.attempts, 0);
17098    }
17099
17100    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
17101        ask_test_home();
17102        let tmp = tempfile::tempdir().unwrap();
17103        let repo = tmp.path().join("repo");
17104        std::fs::create_dir(&repo).unwrap();
17105        init_repo(&repo);
17106        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17107        let origin = tmp.path().join("origin.git");
17108        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
17109            .await
17110            .unwrap();
17111        git::git(
17112            &repo,
17113            &["remote", "add", "origin", origin.to_str().unwrap()],
17114        )
17115        .await
17116        .unwrap();
17117        git::git(&repo, &["checkout", "-b", "fix/retained"])
17118            .await
17119            .unwrap();
17120        std::fs::write(repo.join("retained"), "retained change").unwrap();
17121        git::git(&repo, &["add", "."]).await.unwrap();
17122        git::git(&repo, &["commit", "-m", "retained implementation"])
17123            .await
17124            .unwrap();
17125        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
17126        git::git(&repo, &["checkout", "main"]).await.unwrap();
17127        std::fs::write(repo.join("base-change"), "new base").unwrap();
17128        git::git(&repo, &["add", "."]).await.unwrap();
17129        git::git(&repo, &["commit", "-m", "base moved"])
17130            .await
17131            .unwrap();
17132        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17133        git::git(&repo, &["checkout", "fix/retained"])
17134            .await
17135            .unwrap();
17136        let mut runner = runner_at(RunStatus::Voting);
17137        runner.state.repo = repo.clone();
17138        runner.state.base_commit = base;
17139        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17140        runner.state.config.disk.min_free_bytes = 0;
17141        runner.state.config.graph.review_rounds = 1;
17142        runner.state.config.verify.e2e = vec!["true".into()];
17143        runner.state.config.verify.gate = vec!["true".into()];
17144        runner.state.candidates = vec![Candidate {
17145            index: 0,
17146            label: 'A',
17147            agent: "fixture".into(),
17148            branch: "fix/retained".into(),
17149            worktree: repo.clone(),
17150            summary: String::new(),
17151            stat: String::new(),
17152            files: 1,
17153            commits: 1,
17154            empty: false,
17155            failed: None,
17156            verified_noop: None,
17157            duration_ms: 0,
17158            folded: false,
17159        }];
17160        runner.state.tally = Some(Tally {
17161            first_choice: BTreeMap::new(),
17162            borda: BTreeMap::new(),
17163            winner: 'A',
17164            rankings: 0,
17165            unanimous_initial: true,
17166            deliberated: false,
17167            changed_votes: 0,
17168            unanimous_final: true,
17169            tie_break: None,
17170            judges: 0,
17171            present: 0,
17172            quorum: 0,
17173            met_quorum: true,
17174            uncontested: Some("review-only".into()),
17175        });
17176        let mut task = crate::queue::Task::new(
17177            "fixture".into(),
17178            "fixture".into(),
17179            repo,
17180            crate::queue::Source::Human,
17181        );
17182        task.link_run(&runner.state.id);
17183        crate::queue::Queue::open().put(&mut task).unwrap();
17184        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
17185        runner.state.base_sync = Some(BaseSync {
17186            tip: String::new(),
17187            behind: 1,
17188            attempts: BASE_SYNC_ROUNDS,
17189            conflict: None,
17190            already_in: None,
17191        });
17192        runner.sync_to_base().await.unwrap();
17193        assert_eq!(runner.state.status, RunStatus::Blocked);
17194        (tmp, runner, head, task.id)
17195    }
17196
17197    #[tokio::test]
17198    async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
17199        let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
17200        runner.state.status = RunStatus::Voting;
17201        runner.state.base_sync = None;
17202        runner.state.base_sync_recovery = Default::default();
17203        let scratch = runner.state.dir().join("base-sync");
17204        std::fs::create_dir_all(&scratch).unwrap();
17205        std::fs::write(scratch.join("obstruction"), "keep").unwrap();
17206        runner.sync_to_base().await.unwrap();
17207        assert!(runner.state.resource_block.is_some());
17208        assert!(!runner.state.base_sync_blocked());
17209        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
17210        assert_eq!(runner.state.base_episode_used(), 0);
17211        assert!(runner.state.base_sync_recovery.operations.is_empty());
17212        assert!(runner.state.rebase_fixes.is_empty());
17213        assert_eq!(
17214            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17215            head
17216        );
17217        std::fs::remove_dir_all(&scratch).unwrap();
17218        runner.state.resource_block = None;
17219        runner.sync_to_base().await.unwrap();
17220        assert!(!runner.state.base_sync_blocked());
17221        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17222        assert_eq!(runner.state.base_episode_used(), 1);
17223        assert!(runner.state.base_sync_recovery.operations[0].finished);
17224        assert!(runner.state.rebase_fixes.is_empty());
17225        assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
17226        assert!(git::is_clean(&runner.state.repo).await.unwrap());
17227    }
17228
17229    #[cfg(unix)]
17230    #[tokio::test]
17231    async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
17232        for routed in [false, true] {
17233            let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
17234            let home = crate::run::home();
17235            let queue = crate::queue::Queue::open();
17236            let task = queue.get(&owner).unwrap();
17237            runner.state.task_instruction = Some(task.instruction.clone());
17238            runner.state.driver_exited = true;
17239            runner.state.save().unwrap();
17240            let questions = crate::ask::Questions::at(home.join("questions"));
17241            let mut q = crate::ask::Question::new(
17242                runner.state.id.clone(),
17243                "base-recovery".into(),
17244                "owner".into(),
17245                "Recover bounded base movement?".into(),
17246                String::new(),
17247                vec!["recover".into()],
17248            );
17249            q.actions.insert(
17250                "recover".into(),
17251                crate::ask::ChoiceAction::RecoverBase {
17252                    run: runner.state.id.clone(),
17253                    stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
17254                    episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
17255                },
17256            );
17257            q.routed_task = Some(owner.clone());
17258            q.answer(crate::ask::Answer::Choice("recover".into()))
17259                .unwrap();
17260            questions.put(&mut q).unwrap();
17261            crate::base_recovery_control::recover(
17262                &queue,
17263                &home,
17264                crate::base_recovery_control::Request {
17265                    run: runner.state.id.clone(),
17266                    head: head.clone(),
17267                    actor: "decisions".into(),
17268                    request: q.id.clone(),
17269                    decision: Some((owner.clone(), q.answered_at.unwrap())),
17270                    expected_stop: Some((
17271                        runner.state.base_sync_recovery.stop.clone().unwrap(),
17272                        runner.state.base_sync_recovery.episodes.last().unwrap().id,
17273                    )),
17274                },
17275            )
17276            .await
17277            .unwrap();
17278            if routed {
17279                questions
17280                    .update(&q.id, |q| {
17281                        q.routed_task = Some("another-owner".into());
17282                        Ok(())
17283                    })
17284                    .unwrap();
17285            } else {
17286                let mut task = queue.get(&owner).unwrap();
17287                task.hold_manual(Some("new owner decision".into()));
17288                // Keep the generation to exercise the separate owner-decision binding.
17289                task.status = crate::queue::TaskStatus::Queued;
17290                queue.put(&mut task).unwrap();
17291            }
17292            runner.state = RunState::load(&runner.state.id).unwrap();
17293            runner.sync_to_base().await.unwrap();
17294            let approval = runner
17295                .state
17296                .base_sync_recovery
17297                .authorizations
17298                .last()
17299                .unwrap();
17300            assert!(approval.consumed);
17301            assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17302                "routing"
17303            } else {
17304                "owner decision"
17305            }));
17306            assert_eq!(
17307                runner.state.base_sync.as_ref().unwrap().attempts,
17308                BASE_SYNC_ROUNDS
17309            );
17310            assert_eq!(
17311                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17312                head
17313            );
17314        }
17315    }
17316
17317    #[cfg(unix)]
17318    #[tokio::test]
17319    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17320        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17321        if std::env::var_os(CHILD).is_none() {
17322            let home = tempfile::tempdir().unwrap();
17323            let output = std::process::Command::new(std::env::current_exe().unwrap())
17324                .args([
17325                    "--exact",
17326                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17327                    "--nocapture",
17328                ])
17329                .env(CHILD, "1")
17330                .env("MAGI_HOME", home.path())
17331                .env("MAGI_WORKER_HOME", home.path())
17332                .env("MAGI_WORKER_ID", "abcdef")
17333                .output()
17334                .unwrap();
17335            assert!(
17336                output.status.success()
17337                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17338                "{}\n{}",
17339                String::from_utf8_lossy(&output.stdout),
17340                String::from_utf8_lossy(&output.stderr)
17341            );
17342            return;
17343        }
17344        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17345        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17346        let home = crate::run::home();
17347        let queue = crate::queue::Queue::open();
17348        let mut owner = queue.get(&task).unwrap();
17349        owner.release_operator("test-release", None);
17350        owner.start(runner.state.id.clone());
17351        queue.put(&mut owner).unwrap();
17352        runner.state.driver_exited = true;
17353        runner.state.task_instruction = Some(owner.instruction.clone());
17354        runner.state.save().unwrap();
17355        let _claim = queue.claim(&task).unwrap();
17356        let reserve = |id: &str| {
17357            let directory = home.join("workers/jobs").join(id);
17358            std::fs::create_dir_all(&directory).unwrap();
17359            let job = crate::worker::Job {
17360                id: id.into(),
17361                sequence: 1,
17362                kind: "task".into(),
17363                key: format!("task:{task}"),
17364                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17365                generation: "fixture-generation".into(),
17366                version: None,
17367                status: crate::worker::Status::Running,
17368                result: None,
17369                pid: Some(std::process::id()),
17370                created_at: 0,
17371                finished_at: None,
17372            };
17373            std::fs::write(
17374                directory.join("job.json"),
17375                serde_json::to_vec(&job).unwrap(),
17376            )
17377            .unwrap();
17378            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17379        };
17380        let _own = reserve("abcdef");
17381        let other = reserve("fedcba");
17382        assert!(
17383            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17384                .await
17385                .is_err()
17386        );
17387        assert!(
17388            RunState::load(&runner.state.id)
17389                .unwrap()
17390                .base_sync_recovery
17391                .authorizations
17392                .is_empty()
17393        );
17394        drop(other);
17395        assert!(crate::daemon::is_working_on(
17396            &home,
17397            &runner.state.id,
17398            Timestamp::now()
17399        ));
17400        assert!(
17401            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17402        );
17403        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17404            .await
17405            .unwrap();
17406        assert!(
17407            RunState::load(&runner.state.id)
17408                .unwrap()
17409                .base_recovery_pending()
17410        );
17411    }
17412
17413    #[cfg(unix)]
17414    #[tokio::test]
17415    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17416        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17417        let home = crate::run::home();
17418        runner.state.driver_pid = Some(std::process::id());
17419        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17420        runner.state.driver_exited = false;
17421        runner.state.begin_pass();
17422        runner.state.save().unwrap();
17423        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17424        driver.record(&runner.state).unwrap();
17425        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17426        // Preserve the durable claim as if the process died before Drop saved.
17427        drop(driver);
17428        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17429        assert_eq!(
17430            runner.state.liveness_under(&home, false),
17431            crate::run::Liveness::Dead
17432        );
17433        let queue = crate::queue::Queue::open();
17434        let _claim = queue.claim(&task).unwrap();
17435        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17436            .await
17437            .unwrap();
17438        let disk = RunState::load(&runner.state.id).unwrap();
17439        assert!(!disk.driver_exited);
17440        assert!(disk.base_recovery_pending());
17441    }
17442
17443    #[cfg(unix)]
17444    #[tokio::test]
17445    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17446        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17447        if std::env::var_os(CHILD).is_none() {
17448            use std::os::unix::fs::PermissionsExt;
17449            let tmp = tempfile::tempdir().unwrap();
17450            let gh = tmp.path().join("gh");
17451            std::fs::write(&gh, r#"#!/bin/sh
17452case "$1 $2" in
17453  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17454  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17455  *) exit 2 ;;
17456esac
17457"#).unwrap();
17458            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17459            let mut paths = vec![tmp.path().to_path_buf()];
17460            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17461            let output = std::process::Command::new(std::env::current_exe().unwrap())
17462                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17463                .env(CHILD, "1")
17464                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17465                .env("PATH", std::env::join_paths(paths).unwrap())
17466                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17467                .output().unwrap();
17468            assert!(
17469                output.status.success()
17470                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17471                "{}\n{}",
17472                String::from_utf8_lossy(&output.stdout),
17473                String::from_utf8_lossy(&output.stderr)
17474            );
17475            return;
17476        }
17477        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17478        let repo = runner.state.repo.clone();
17479        git::git(&repo, &["checkout", "main"]).await.unwrap();
17480        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17481        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17482        git::git(&repo, &["checkout", "fix/retained"])
17483            .await
17484            .unwrap();
17485        runner.state.pr = Some(crate::run::PrRecord {
17486            url: "https://example.test/project/pull/1".into(),
17487            number: 1,
17488            state: "open".into(),
17489            checks: "unknown".into(),
17490            round: 0,
17491            rounds: 1,
17492            red_at_merge: Vec::new(),
17493        });
17494        runner.sync_to_base().await.unwrap();
17495        assert_eq!(runner.state.status, RunStatus::Blocked);
17496        assert_eq!(
17497            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17498            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17499        );
17500        assert!(runner.state.base_sync_closure_pending());
17501        assert!(crate::handover::can_resume(&runner.state));
17502        let queue = crate::queue::Queue::open();
17503        assert!(!crate::base_sync_recovery::held_task(
17504            &queue,
17505            &queue.get(&task).unwrap()
17506        ));
17507        assert!(runner.state.authorize_base_sync(&task).is_err());
17508        // Legacy closure stops have no typed stop but must retain the checkpoint.
17509        runner.state.base_sync_recovery.stop = None;
17510        runner.state.status = RunStatus::Voting;
17511        runner.state.gate_ran = true;
17512        runner.state.config.graph.review_rounds = 0;
17513        runner.state.save().unwrap();
17514        assert!(crate::handover::can_resume(&runner.state));
17515        runner.state = RunState::load(&runner.state.id).unwrap();
17516        runner.execute_graph().await.unwrap();
17517        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17518        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17519        assert_eq!(
17520            runner.state.base_sync.as_ref().unwrap().attempts,
17521            BASE_SYNC_ROUNDS
17522        );
17523        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17524        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17525        assert!(runner.state.reviews.is_empty());
17526        assert!(runner.state.gate.is_empty());
17527    }
17528
17529    #[cfg(unix)]
17530    #[tokio::test]
17531    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17532        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17533        runner.state.driver_exited = true;
17534        runner.state.save().unwrap();
17535        let home = crate::run::home();
17536        let queue = crate::queue::Queue::open();
17537        let _claim = queue.claim(&task).unwrap();
17538        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17539        assert!(
17540            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17541                .await
17542                .is_err()
17543        );
17544        drop(lease);
17545        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17546        assert!(
17547            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17548                .await
17549                .is_err()
17550        );
17551        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17552        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17553            .await
17554            .unwrap();
17555        let disk = RunState::load(&runner.state.id).unwrap();
17556        assert!(disk.base_recovery_pending());
17557        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17558        assert!(
17559            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17560                .await
17561                .is_err()
17562        );
17563        assert_eq!(
17564            RunState::load(&runner.state.id)
17565                .unwrap()
17566                .base_sync_recovery
17567                .authorizations
17568                .len(),
17569            1
17570        );
17571        let mut owner = queue.get(&task).unwrap();
17572        owner.runs.push("20260101-000000-ffff".into());
17573        queue.put(&mut owner).unwrap();
17574        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17575            .await
17576            .unwrap_err();
17577        assert!(error.to_string().contains("run task ownership changed"));
17578    }
17579
17580    #[cfg(unix)]
17581    #[tokio::test]
17582    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17583        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17584        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17585        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17586        legacy["status"] = serde_json::json!("voting");
17587        runner.state = serde_json::from_value(legacy).unwrap();
17588        runner.state.save().unwrap();
17589        for _ in 0..3 {
17590            runner.execute_graph().await.unwrap();
17591            assert_eq!(runner.state.status, RunStatus::Blocked);
17592            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17593            assert!(!crate::handover::can_resume(&runner.state));
17594            assert_eq!(
17595                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17596                head
17597            );
17598            assert!(runner.state.reviews.is_empty());
17599            assert!(!runner.state.gate_ran);
17600        }
17601        let notice = crate::notices::run_ended(&runner.state).unwrap();
17602        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17603        runner.state.begin_pass();
17604        runner.state.status = RunStatus::Voting;
17605        runner.deliberate().await.unwrap();
17606        runner.vote().await.unwrap();
17607        runner.finish_after_tally().await.unwrap();
17608        assert_eq!(runner.state.status, RunStatus::Blocked);
17609        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17610        assert_eq!(
17611            crate::notices::run_ended(&runner.state).unwrap().since,
17612            notice.since
17613        );
17614    }
17615
17616    #[cfg(unix)]
17617    #[tokio::test]
17618    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17619        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17620        for detail in [
17621            "fix/retained conflicts with origin/main and did not rebase: conflict",
17622            "rebased fix/retained locally but origin refused the push",
17623            "unknown failure",
17624        ] {
17625            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17626            runner.state.record_base_sync_stop(&head);
17627            assert!(runner.state.authorize_base_sync(&task).is_err());
17628            runner.state.status = RunStatus::Voting;
17629            runner.execute_graph().await.unwrap();
17630            assert_eq!(runner.state.status, RunStatus::Blocked);
17631            assert!(!crate::handover::can_resume(&runner.state));
17632            assert_eq!(
17633                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17634                head
17635            );
17636        }
17637    }
17638
17639    #[cfg(unix)]
17640    #[tokio::test]
17641    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17642        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17643        runner.state.authorize_base_sync(&task).unwrap();
17644        assert!(runner.state.consume_base_recovery(&head));
17645        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17646        runner.state.reserve_base_operation(&head, &tip);
17647        runner.state.save().unwrap();
17648        runner.state = RunState::load(&runner.state.id).unwrap();
17649        assert!(!runner.state.consume_base_recovery(&head));
17650        runner.sync_to_base().await.unwrap();
17651        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17652        assert_eq!(runner.state.base_episode_used(), 1);
17653        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17654        assert!(
17655            runner
17656                .state
17657                .base_sync_recovery
17658                .operations
17659                .last()
17660                .unwrap()
17661                .finished
17662        );
17663        assert!(!runner.state.base_sync_blocked());
17664        runner.sync_to_base().await.unwrap();
17665        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17666    }
17667
17668    #[cfg(unix)]
17669    #[tokio::test]
17670    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17671        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17672        // Both historical diagnostic spellings retain an exhaustion identity.
17673        let original = runner
17674            .state
17675            .base_sync
17676            .as_ref()
17677            .unwrap()
17678            .conflict
17679            .clone()
17680            .unwrap();
17681        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17682            original
17683                .replace("synchronization(s)", "rebase(s)")
17684                .replace("synchronizing", "rebasing"),
17685        );
17686        runner.state.base_sync_recovery = Default::default();
17687        runner.state.record_base_sync_stop(&head);
17688        assert_eq!(
17689            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17690            crate::base_sync_recovery::StopKind::Exhausted
17691        );
17692        runner.state.authorize_base_sync(&task).unwrap();
17693        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17694        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17695            .as_object_mut()
17696            .unwrap();
17697        auth.remove("interval");
17698        auth.remove("scope");
17699        runner.state = serde_json::from_value(legacy).unwrap();
17700        assert!(runner.state.consume_base_recovery(&head));
17701        assert_eq!(runner.state.base_episode_used(), 3);
17702        runner.sync_to_base().await.unwrap();
17703        assert_eq!(runner.state.base_episode_used(), 4);
17704        runner.state.resource_block = Some(crate::run::ResourceBlock {
17705            node: "gate".into(),
17706            kind: "cache".into(),
17707            detail: "legacy wait".into(),
17708            at: Timestamp::now(),
17709            cause: crate::run::ResourceBlockCause::Unrecorded,
17710        });
17711        runner.state.save().unwrap();
17712        assert!(
17713            runner
17714                .state
17715                .base_sync_recovery
17716                .episodes
17717                .last()
17718                .unwrap()
17719                .boundary
17720                .is_none()
17721        );
17722        runner.state.resource_block = None;
17723        advance_base_fixture(&runner).await;
17724        runner.sync_to_base().await.unwrap();
17725        assert!(runner.state.base_sync_blocked());
17726        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17727    }
17728
17729    #[cfg(unix)]
17730    async fn advance_base_fixture(runner: &Runner) {
17731        let repo = &runner.state.repo;
17732        for args in [
17733            vec!["checkout", "main"],
17734            vec!["commit", "--allow-empty", "-m", "base advanced"],
17735            vec!["push", "origin", "main"],
17736            vec!["checkout", "fix/retained"],
17737        ] {
17738            git::git(repo, &args).await.unwrap();
17739        }
17740    }
17741
17742    #[cfg(unix)]
17743    fn fresh_base_interval_fixture(runner: &mut Runner) {
17744        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17745        runner.state.base_sync.as_mut().unwrap().conflict = None;
17746        runner.state.base_sync_recovery = Default::default();
17747        runner.state.status = RunStatus::Voting;
17748    }
17749
17750    #[cfg(unix)]
17751    #[tokio::test]
17752    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17753        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17754        fresh_base_interval_fixture(&mut runner);
17755        runner.state.begin_pass();
17756        runner.sync_to_base().await.unwrap();
17757        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17758        runner.state.resource_block = Some(crate::run::ResourceBlock {
17759            node: "gate".into(),
17760            kind: "build cache".into(),
17761            detail: "busy".into(),
17762            at: Timestamp::now(),
17763            cause: crate::run::ResourceBlockCause::Timeout {
17764                waited_secs: 1,
17765                limit_secs: 1,
17766            },
17767        });
17768        runner.state.save().unwrap();
17769        let queue = crate::queue::Queue::open();
17770        let mut owner = queue.get(&task).unwrap();
17771        runner.state.task_instruction = Some(owner.instruction.clone());
17772        owner.release_operator("fixture-owner", None);
17773        queue.put(&mut owner).unwrap();
17774        let approval = owner.resumption.clone().unwrap();
17775        runner.state.resource_block = None;
17776        runner.state.save().unwrap();
17777        runner.state = RunState::load(&runner.state.id).unwrap();
17778        advance_base_fixture(&runner).await;
17779        runner.sync_to_base().await.unwrap();
17780        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17781        assert_eq!(runner.state.base_episode_used(), 1);
17782        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17783        assert!(
17784            runner.state.base_sync_recovery.episodes[0]
17785                .boundary
17786                .as_ref()
17787                .unwrap()
17788                .consumed
17789        );
17790        assert!(
17791            !runner
17792                .state
17793                .renew_base_episode(&task, &approval, &head, "base")
17794        );
17795        for _ in 0..3 {
17796            owner.release_operator("fixture-owner", None);
17797            queue.put(&mut owner).unwrap();
17798            runner.state = RunState::load(&runner.state.id).unwrap();
17799            runner.state.begin_pass();
17800            runner.sync_to_base().await.unwrap();
17801            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17802            assert_eq!(runner.state.base_episode_used(), 1);
17803        }
17804        runner.state.resource_block = Some(crate::run::ResourceBlock {
17805            node: "gate".into(),
17806            kind: "disk pressure".into(),
17807            detail: "free space required".into(),
17808            at: Timestamp::now(),
17809            cause: crate::run::ResourceBlockCause::Other,
17810        });
17811        runner.state.end_pass_parked(ParkCause::Interrupt {
17812            reason: "disk pressure".into(),
17813        });
17814        runner.state.save().unwrap();
17815        owner.release_operator("fixture-owner-after-disk", None);
17816        queue.put(&mut owner).unwrap();
17817        runner.state.resource_block = None;
17818        advance_base_fixture(&runner).await;
17819        runner.sync_to_base().await.unwrap();
17820        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17821        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17822        assert_eq!(runner.state.base_episode_used(), 1);
17823        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17824    }
17825
17826    #[cfg(unix)]
17827    #[tokio::test]
17828    async fn base_recovery_continuous_advancement_stops_at_four() {
17829        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17830        fresh_base_interval_fixture(&mut runner);
17831        for spent in 1..=BASE_SYNC_ROUNDS {
17832            runner.sync_to_base().await.unwrap();
17833            assert!(!runner.state.base_sync_blocked());
17834            assert_eq!(runner.state.base_episode_used(), spent);
17835            advance_base_fixture(&runner).await;
17836        }
17837        runner.sync_to_base().await.unwrap();
17838        assert_eq!(
17839            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17840            crate::base_sync_recovery::StopKind::Exhausted
17841        );
17842        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17843        assert!(stop.unverified_sync);
17844        let action = runner.state.base_sync_next_action();
17845        assert!(action.contains("never reached its gate"));
17846        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17847        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17848        runner.state = RunState::load(&runner.state.id).unwrap();
17849        runner.sync_to_base().await.unwrap();
17850        assert_eq!(runner.state.base_episode_used(), 4);
17851        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17852    }
17853
17854    #[cfg(unix)]
17855    #[tokio::test]
17856    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17857        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17858        runner.state.ensure_base_episode();
17859        // Verification progress alone is not evidence of a successful sync.
17860        runner.state.record_base_progress(&head);
17861        runner.state.base_sync_recovery.stop = None;
17862        runner.state.record_base_sync_stop(&head);
17863        assert!(
17864            !runner
17865                .state
17866                .base_sync_recovery
17867                .stop
17868                .as_ref()
17869                .unwrap()
17870                .unverified_sync
17871        );
17872
17873        runner.state.finish_base_operation(Some(&head));
17874        // A gate for another head cannot verify this synchronization.
17875        runner.state.gate_ran = true;
17876        runner.state.gate_head = Some("older-head".into());
17877        runner.state.base_sync_recovery.stop = None;
17878        runner.state.record_base_sync_stop(&head);
17879        runner.state.save().unwrap();
17880        runner.state = RunState::load(&runner.state.id).unwrap();
17881        assert!(
17882            runner
17883                .state
17884                .base_sync_recovery
17885                .stop
17886                .as_ref()
17887                .unwrap()
17888                .unverified_sync
17889        );
17890
17891        runner.state.gate_head = Some(head.clone());
17892        runner.state.base_sync_recovery.stop = None;
17893        runner.state.record_base_sync_stop(&head);
17894        assert!(
17895            !runner
17896                .state
17897                .base_sync_recovery
17898                .stop
17899                .as_ref()
17900                .unwrap()
17901                .unverified_sync
17902        );
17903        assert!(
17904            !runner
17905                .state
17906                .base_sync_next_action()
17907                .contains("never reached its gate")
17908        );
17909
17910        // Historical records cannot establish the missing synchronization evidence.
17911        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17912        legacy["base_sync_recovery"]["stop"]
17913            .as_object_mut()
17914            .unwrap()
17915            .remove("unverified_sync");
17916        legacy["base_sync_recovery"]["episodes"][0]
17917            .as_object_mut()
17918            .unwrap()
17919            .remove("last_sync_head");
17920        runner.state = serde_json::from_value(legacy).unwrap();
17921        assert!(
17922            !runner
17923                .state
17924                .base_sync_recovery
17925                .stop
17926                .as_ref()
17927                .unwrap()
17928                .unverified_sync
17929        );
17930        assert!(
17931            runner.state.base_sync_recovery.episodes[0]
17932                .last_sync_head
17933                .is_none()
17934        );
17935    }
17936
17937    #[cfg(unix)]
17938    #[tokio::test]
17939    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17940        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17941        runner.state.authorize_base_sync(&task).unwrap();
17942        assert!(
17943            runner
17944                .state
17945                .base_sync_next_action()
17946                .contains("Approval recorded")
17947        );
17948        assert!(
17949            runner
17950                .state
17951                .base_sync_next_action()
17952                .contains("4/4 in episode 0, 4 lifetime")
17953        );
17954        assert!(runner.state.consume_base_recovery(&head));
17955        assert!(
17956            runner
17957                .state
17958                .base_sync_next_action()
17959                .contains("0/4 in episode 1, 4 lifetime")
17960        );
17961    }
17962
17963    #[cfg(unix)]
17964    #[tokio::test]
17965    async fn base_recovery_changed_head_invalidates_without_spending() {
17966        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17967        runner.state.authorize_base_sync(&task).unwrap();
17968        git::git(
17969            &runner.state.repo,
17970            &["commit", "--allow-empty", "-m", "external head change"],
17971        )
17972        .await
17973        .unwrap();
17974        runner.sync_to_base().await.unwrap();
17975        let a = runner
17976            .state
17977            .base_sync_recovery
17978            .authorizations
17979            .last()
17980            .unwrap();
17981        assert!(a.consumed);
17982        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17983        runner.state = RunState::load(&runner.state.id).unwrap();
17984        assert!(
17985            runner
17986                .state
17987                .base_sync_next_action()
17988                .contains("Approval invalidated")
17989        );
17990        assert!(!runner.state.base_recovery_pending());
17991        assert!(runner.state.base_sync_blocked());
17992        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17993    }
17994
17995    #[cfg(unix)]
17996    #[tokio::test]
17997    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17998        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17999        runner.state.driver_exited = true;
18000        runner.state.save().unwrap();
18001        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18002            .await
18003            .unwrap();
18004        runner.state = RunState::load(&runner.state.id).unwrap();
18005        advance_base_fixture(&runner).await;
18006        runner.sync_to_base().await.unwrap();
18007        let a = runner
18008            .state
18009            .base_sync_recovery
18010            .authorizations
18011            .last()
18012            .unwrap();
18013        assert!(
18014            a.consumed
18015                && a.invalidated
18016                    .as_deref()
18017                    .unwrap()
18018                    .contains("fetched base changed")
18019        );
18020        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18021        assert_eq!(
18022            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18023            head
18024        );
18025        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18026            .await
18027            .unwrap();
18028        runner.state = RunState::load(&runner.state.id).unwrap();
18029        runner.sync_to_base().await.unwrap();
18030        assert!(!runner.state.base_sync_blocked());
18031        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18032    }
18033
18034    #[cfg(unix)]
18035    #[tokio::test]
18036    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
18037        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18038        runner.state.authorize_base_sync(&task).unwrap();
18039        assert!(runner.state.consume_base_recovery(&head));
18040        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18041        runner.state.reserve_base_operation(&head, &tip);
18042        runner.state.save().unwrap();
18043        let scratch = runner.state.dir().join("base-sync");
18044        assert_eq!(
18045            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18046                .await
18047                .unwrap(),
18048            crate::rebase::Rebased::Applied
18049        );
18050        runner.state = RunState::load(&runner.state.id).unwrap();
18051        runner.sync_to_base().await.unwrap();
18052        assert!(
18053            runner
18054                .state
18055                .base_sync_recovery
18056                .operations
18057                .last()
18058                .unwrap()
18059                .finished
18060        );
18061        assert_eq!(runner.state.base_episode_used(), 1);
18062        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18063        assert!(!runner.state.base_sync_blocked());
18064    }
18065
18066    #[cfg(unix)]
18067    #[tokio::test]
18068    async fn base_recovery_crash_preserves_untracked_files() {
18069        for ignored in [false, true] {
18070            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18071            runner.state.authorize_base_sync(&task).unwrap();
18072            assert!(runner.state.consume_base_recovery(&head));
18073            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18074            runner.state.reserve_base_operation(&head, &tip);
18075            runner.state.save().unwrap();
18076            let scratch = runner.state.dir().join("base-sync");
18077            assert_eq!(
18078                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18079                    .await
18080                    .unwrap(),
18081                crate::rebase::Rebased::Applied
18082            );
18083            let repo = &runner.state.repo;
18084            let result = git::rev_parse(repo, "HEAD").await.unwrap();
18085            assert_ne!(result, head);
18086            if ignored {
18087                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
18088                    .await
18089                    .unwrap();
18090                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
18091            }
18092            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
18093            assert!(
18094                git::git_raw(repo, &["diff", "--quiet", &head])
18095                    .await
18096                    .unwrap()
18097                    .ok()
18098            );
18099            // Resume after the branch moved, before its checkout was reset.
18100            runner.state = RunState::load(&runner.state.id).unwrap();
18101            runner.sync_to_base().await.unwrap();
18102            assert!(runner.state.base_sync_blocked());
18103            assert!(!runner.state.base_sync_recovery.operations[0].finished);
18104            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18105            assert_eq!(
18106                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18107                result
18108            );
18109            assert_eq!(
18110                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
18111                "retained work\n"
18112            );
18113            assert!(
18114                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
18115                    .await
18116                    .unwrap()
18117                    .ok()
18118            );
18119        }
18120    }
18121
18122    #[cfg(unix)]
18123    #[tokio::test]
18124    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
18125        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18126        runner.state.authorize_base_sync(&task).unwrap();
18127        assert!(runner.state.consume_base_recovery(&head));
18128        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18129        runner.state.reserve_base_operation(&head, &tip);
18130        runner.state.save().unwrap();
18131        let scratch = runner.state.dir().join("base-sync");
18132        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18133            .await
18134            .unwrap();
18135        git::sync_to_head(&runner.state.repo).await.unwrap();
18136        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
18137        git::git(&runner.state.repo, &["add", "foreign"])
18138            .await
18139            .unwrap();
18140        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
18141            .await
18142            .unwrap();
18143        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18144        runner.state = RunState::load(&runner.state.id).unwrap();
18145        runner.sync_to_base().await.unwrap();
18146        assert!(runner.state.base_sync_blocked());
18147        assert!(
18148            !runner
18149                .state
18150                .base_sync_recovery
18151                .operations
18152                .last()
18153                .unwrap()
18154                .finished
18155        );
18156        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18157        assert_eq!(
18158            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18159            foreign_head
18160        );
18161    }
18162
18163    #[cfg(unix)]
18164    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
18165    {
18166        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18167        fresh_base_interval_fixture(&mut runner);
18168        let repo = &runner.state.repo;
18169        git::git(repo, &["checkout", "main"]).await.unwrap();
18170        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
18171        git::git(repo, &["add", "retained"]).await.unwrap();
18172        git::git(repo, &["commit", "-m", "upstream retained change"])
18173            .await
18174            .unwrap();
18175        git::git(repo, &["push", "origin", "main"]).await.unwrap();
18176        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
18177        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
18178        let scratch = runner.state.dir().join("base-sync");
18179        assert!(matches!(
18180            git::rebase_start(repo, &scratch, "fix/retained", &tip)
18181                .await
18182                .unwrap(),
18183            git::RebaseStart::Conflicted(_)
18184        ));
18185        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
18186            agent: "fixture".into(),
18187            paths: vec!["retained".into()],
18188            from: Some(head.clone()),
18189            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
18190            index_outside: None,
18191            finished: false,
18192            error: None,
18193            launcher: None,
18194            infra: None,
18195            outage: None,
18196        });
18197        (tmp, runner, head, tip, scratch)
18198    }
18199
18200    #[cfg(unix)]
18201    #[tokio::test]
18202    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
18203        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18204        // A resource stop from before reservations existed, with resolved
18205        // contents left for the launcher's ordinary staging continuation.
18206        std::fs::write(
18207            scratch.join("retained"),
18208            "upstream and retained resolution\n",
18209        )
18210        .unwrap();
18211        runner.state.resource_block = Some(crate::run::ResourceBlock {
18212            node: "rebase".into(),
18213            kind: "repository coordination".into(),
18214            detail: "busy".into(),
18215            at: Timestamp::now(),
18216            cause: crate::run::ResourceBlockCause::Other,
18217        });
18218        runner.state.save().unwrap();
18219        // Advance the remote without moving either checkout participating
18220        // in the standing rebase.
18221        let newer = git::git(
18222            &runner.state.repo,
18223            &[
18224                "commit-tree",
18225                "main^{tree}",
18226                "-p",
18227                "main",
18228                "-m",
18229                "base advanced during wait",
18230            ],
18231        )
18232        .await
18233        .unwrap();
18234        git::git(
18235            &runner.state.repo,
18236            &[
18237                "push",
18238                "origin",
18239                &format!("{}:refs/heads/main", newer.trim()),
18240            ],
18241        )
18242        .await
18243        .unwrap();
18244        runner.state = RunState::load(&runner.state.id).unwrap();
18245        runner.state.resource_block = None;
18246        runner.sync_to_base().await.unwrap();
18247        assert!(!runner.state.base_sync_blocked());
18248        let op = &runner.state.base_sync_recovery.operations[0];
18249        assert_eq!(op.head, head);
18250        assert_eq!(op.tip, tip);
18251        assert!(op.finished);
18252        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18253        assert_eq!(runner.state.rebase_fixes.len(), 1);
18254        let latest = git::rev_parse(&runner.state.repo, "origin/main")
18255            .await
18256            .unwrap();
18257        assert_ne!(latest, tip);
18258        runner.state = RunState::load(&runner.state.id).unwrap();
18259        runner.sync_to_base().await.unwrap();
18260        assert!(!runner.state.base_sync_blocked());
18261        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
18262        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
18263        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18264    }
18265
18266    #[cfg(unix)]
18267    #[tokio::test]
18268    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
18269        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18270        runner.state.reserve_base_operation(&head, &tip);
18271        runner.state.save().unwrap();
18272        std::fs::write(
18273            scratch.join("retained"),
18274            "upstream and retained resolution\n",
18275        )
18276        .unwrap();
18277        git::git(&scratch, &["add", "retained"]).await.unwrap();
18278        git::git(
18279            &scratch,
18280            &["-c", "core.editor=true", "rebase", "--continue"],
18281        )
18282        .await
18283        .unwrap();
18284        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18285        assert!(
18286            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18287                .await
18288                .unwrap()
18289                .is_empty()
18290        );
18291        assert_eq!(
18292            crate::rebase::finish(
18293                &mut runner.state,
18294                &scratch,
18295                "fix/retained",
18296                &head,
18297                &tip,
18298                &["retained".into()],
18299                ""
18300            )
18301            .await
18302            .unwrap(),
18303            crate::rebase::Rebased::Applied
18304        );
18305        assert!(!scratch.exists());
18306        // Simulate losing the caller before finish_base_operation/save.
18307        runner.state = RunState::load(&runner.state.id).unwrap();
18308        let op = &runner.state.base_sync_recovery.operations[0];
18309        assert!(!op.finished);
18310        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18311        runner.sync_to_base().await.unwrap();
18312        assert!(!runner.state.base_sync_blocked());
18313        assert!(runner.state.base_sync_recovery.operations[0].finished);
18314        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18315        assert_eq!(runner.state.rebase_fixes.len(), 1);
18316        let winner = runner.state.winner().unwrap().clone();
18317        let wait = runner.repository_wait();
18318        assert!(
18319            verify_current_head(&mut runner.state, &winner, &wait)
18320                .await
18321                .unwrap()
18322        );
18323        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18324    }
18325
18326    #[cfg(unix)]
18327    #[tokio::test]
18328    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18329        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18330            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18331            runner.state.reserve_base_operation(&head, &tip);
18332            runner.state.save().unwrap();
18333            std::fs::write(
18334                scratch.join("retained"),
18335                "upstream and retained resolution\n",
18336            )
18337            .unwrap();
18338            git::git(&scratch, &["add", "retained"]).await.unwrap();
18339            git::git(
18340                &scratch,
18341                &["-c", "core.editor=true", "rebase", "--continue"],
18342            )
18343            .await
18344            .unwrap();
18345            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18346            assert!(!git::rebase_in_progress(&scratch).await);
18347            assert!(
18348                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18349                    .await
18350                    .unwrap()
18351                    .is_empty()
18352            );
18353            match damage {
18354                Some("record") => {
18355                    let path =
18356                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18357                            .await
18358                            .unwrap();
18359                    std::fs::write(scratch.join(path), "").unwrap();
18360                }
18361                Some("foreign") => {
18362                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18363                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
18364                    git::git(&scratch, &["commit", "-m", "foreign work"])
18365                        .await
18366                        .unwrap();
18367                }
18368                Some("dirty") => {
18369                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18370                }
18371                _ => {}
18372            }
18373            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18374            // Crash immediately after Git continuation, before finish runs.
18375            runner.state = RunState::load(&runner.state.id).unwrap();
18376            assert!(
18377                runner.state.base_sync_recovery.operations[0]
18378                    .validated_head
18379                    .is_none()
18380            );
18381            runner.sync_to_base().await.unwrap();
18382            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18383            assert_eq!(runner.state.base_episode_used(), 1);
18384            assert_eq!(runner.state.rebase_fixes.len(), 1);
18385            assert_eq!(
18386                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18387                retained_head
18388            );
18389            if damage.is_some() {
18390                assert!(runner.state.base_sync_blocked());
18391                assert!(scratch.exists());
18392                assert!(!runner.state.base_sync_recovery.operations[0].finished);
18393                assert!(
18394                    runner.state.base_sync_recovery.operations[0]
18395                        .validated_head
18396                        .is_none()
18397                );
18398            } else {
18399                assert!(!runner.state.base_sync_blocked());
18400                assert!(!scratch.exists());
18401                assert!(runner.state.base_sync_recovery.operations[0].finished);
18402                assert_eq!(
18403                    runner.state.base_sync_recovery.operations[0]
18404                        .validated_head
18405                        .as_deref(),
18406                    Some(result.as_str())
18407                );
18408                let winner = runner.state.winner().unwrap().clone();
18409                let wait = runner.repository_wait();
18410                assert!(
18411                    verify_current_head(&mut runner.state, &winner, &wait)
18412                        .await
18413                        .unwrap()
18414                );
18415                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18416                runner.state = RunState::load(&runner.state.id).unwrap();
18417                runner.sync_to_base().await.unwrap();
18418                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18419            }
18420        }
18421    }
18422
18423    #[cfg(unix)]
18424    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18425        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18426        // This is an ordinary gate stop, not an authorized base-budget recovery.
18427        runner.state.prep_complete = Some(true);
18428        runner.state.base_sync = None;
18429        runner.state.base_sync_recovery = Default::default();
18430        runner.state.config.graph.gate_fix_rounds = 1;
18431        let mut review = review_round(true, 0, 1, 1, false, true);
18432        review.head = head.clone();
18433        review.verified_head = Some(head.clone());
18434        runner.state.reviews = vec![review];
18435        runner.state.gate = vec![CommandOutcome {
18436            command: "old gate".into(),
18437            code: Some(1),
18438            output_tail: "old failure".into(),
18439            duration_ms: 0,
18440            resource_blocked: false,
18441        }];
18442        runner.state.gate_head = Some(head.clone());
18443        runner.state.gate_ran = true;
18444        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18445            agent: "fixture".into(),
18446            failed: runner.state.gate.clone(),
18447            notes: "spent".into(),
18448            committed: false,
18449            error: None,
18450        }];
18451        runner.state.land_approval = Some(crate::run::LandApproval {
18452            question: "old approval".into(),
18453            head: head.clone(),
18454        });
18455        runner.state.status = RunStatus::Blocked;
18456        (tmp, runner, head)
18457    }
18458
18459    #[cfg(unix)]
18460    #[tokio::test]
18461    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18462        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18463            let (tmp, mut runner, old) = resumed_gate_fixture().await;
18464            let phases = tmp.path().join("phases");
18465            runner.state.config.verify.e2e = vec![format!(
18466                "echo e2e >> '{}'; {}",
18467                phases.display(),
18468                if e2e_ok {
18469                    "true"
18470                } else {
18471                    "echo new-e2e-failure; false"
18472                }
18473            )];
18474            runner.state.config.verify.gate = vec![format!(
18475                "echo gate >> '{}'; {}",
18476                phases.display(),
18477                if gate_ok {
18478                    "true"
18479                } else {
18480                    "echo new-gate-failure; false"
18481                }
18482            )];
18483            runner.state.save().unwrap();
18484            runner.state = RunState::load(&runner.state.id).unwrap();
18485            runner.sync_to_base().await.unwrap();
18486            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18487            assert_ne!(head, old);
18488            // Reentry after Git updated but before verification was invalidated.
18489            runner.state = RunState::load(&runner.state.id).unwrap();
18490            runner.execute_graph().await.unwrap();
18491            assert_eq!(runner.state.gate_fixes.len(), 1);
18492            assert_eq!(runner.state.reviews.len(), 1);
18493            assert_eq!(runner.state.reviews[0].head, old);
18494            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18495            assert_eq!(runner.state.verification_history.len(), 1);
18496            let history = &runner.state.verification_history[0];
18497            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18498            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18499            assert!(history.archived_at.is_some());
18500            assert_eq!(history.fixes_spent, Some(1));
18501            assert_eq!(history.gate[0].output_tail, "old failure");
18502            let archive_events: Vec<_> = runner
18503                .state
18504                .events
18505                .iter()
18506                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18507                .collect();
18508            assert_eq!(archive_events.len(), 1);
18509            assert!(archive_events[0].message.contains(&short(&old)));
18510            assert!(archive_events[0].message.contains(&short(&head)));
18511            assert!(
18512                archive_events[0]
18513                    .message
18514                    .contains("1 gate-fix round(s) already spent stay spent")
18515            );
18516            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18517            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18518            if e2e_ok {
18519                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18520                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18521            } else {
18522                assert!(!runner.state.gate_ran);
18523            }
18524            assert_eq!(
18525                runner.state.status,
18526                if e2e_ok && gate_ok {
18527                    RunStatus::Ready
18528                } else {
18529                    RunStatus::Blocked
18530                }
18531            );
18532            if !e2e_ok || !gate_ok {
18533                let cause = runner.state.verification_blocking_cause().unwrap();
18534                assert!(cause.contains(&head));
18535                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18536                // A later successful progress event must not become the cause.
18537                runner.state.event("land", "rebased successfully");
18538                let notice = crate::notices::run_ended(&runner.state).unwrap();
18539                assert!(!format!("{notice:?}").contains("rebased successfully"));
18540                assert!(runner.state.merge.is_none());
18541            }
18542            let before = std::fs::read_to_string(&phases).unwrap();
18543            runner.state.save().unwrap();
18544            runner.state = RunState::load(&runner.state.id).unwrap();
18545            runner.review_loop().await.unwrap();
18546            runner.gate().await.unwrap();
18547            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18548            assert_eq!(runner.state.verification_history.len(), 1);
18549            let history = &runner.state.verification_history[0];
18550            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18551            assert!(history.archived_at.is_some());
18552            assert_eq!(history.fixes_spent, Some(1));
18553        }
18554    }
18555
18556    #[cfg(unix)]
18557    #[tokio::test]
18558    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18559        let (tmp, mut runner, head) = resumed_gate_fixture().await;
18560        let marker = tmp.path().join("gate-ran");
18561        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18562        runner.review_loop().await.unwrap();
18563        runner.gate().await.unwrap();
18564        assert_eq!(runner.state.status, RunStatus::Blocked);
18565        assert!(!marker.exists());
18566        assert!(runner.state.verification_history.is_empty());
18567        runner.state.gate_head = None;
18568        runner.gate().await.unwrap();
18569        assert!(marker.exists());
18570        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18571        assert_eq!(runner.state.verification_history.len(), 1);
18572        assert_eq!(runner.state.gate_fixes.len(), 1);
18573    }
18574
18575    #[cfg(unix)]
18576    #[tokio::test]
18577    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18578        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18579        runner.sync_to_base().await.unwrap();
18580        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18581        runner.state.reconcile_verification_head(&head);
18582        // Both crash-before-command and resource wait are unresolved evidence.
18583        for outcomes in [
18584            Vec::new(),
18585            vec![CommandOutcome {
18586                command: "true".into(),
18587                code: None,
18588                output_tail: "cache unavailable".into(),
18589                duration_ms: 0,
18590                resource_blocked: true,
18591            }],
18592        ] {
18593            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18594                head: head.clone(),
18595                passed: false,
18596                outcomes,
18597            });
18598            runner.state.gate_ran = false;
18599            runner.state.gate.clear();
18600            runner.state.save().unwrap();
18601            runner.state = RunState::load(&runner.state.id).unwrap();
18602            runner.merge().await.unwrap();
18603            assert!(runner.state.merge.is_none());
18604            runner.review_loop().await.unwrap();
18605            runner.gate().await.unwrap();
18606            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18607            assert!(runner.state.gate_allows_progress());
18608            assert_eq!(runner.state.gate_fixes.len(), 1);
18609            assert!(runner.state.merge.is_none());
18610        }
18611        // Moving the winner after a successful gate cannot publish an unchecked SHA.
18612        git::git(
18613            &runner.state.repo,
18614            &["commit", "--allow-empty", "-m", "external update"],
18615        )
18616        .await
18617        .unwrap();
18618        runner.merge().await.unwrap();
18619        assert!(runner.state.merge.is_none());
18620        assert_eq!(runner.state.status, RunStatus::Blocked);
18621    }
18622
18623    #[cfg(unix)]
18624    #[tokio::test]
18625    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18626        for waiting_on_gate in [false, true] {
18627            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18628            runner.sync_to_base().await.unwrap();
18629            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18630            runner.state.reconcile_verification_head(&head);
18631            if waiting_on_gate {
18632                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18633                    head: head.clone(),
18634                    passed: true,
18635                    outcomes: Vec::new(),
18636                });
18637            }
18638            let cache = tmp.path().join("cache");
18639            runner.state.config.verify.e2e =
18640                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18641            runner.state.config.verify.gate = vec!["true".into()];
18642            runner.state.config.graph.timeout_resource_wait = 1;
18643            let owner =
18644                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18645            let guard =
18646                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18647                    crate::cache::AcquireOutcome::Acquired(g) => g,
18648                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18649                };
18650            runner.review_loop().await.unwrap();
18651            runner.gate().await.unwrap();
18652            assert!(!runner.state.gate_ran);
18653            assert_eq!(runner.state.gate_fixes.len(), 1);
18654            assert!(runner.state.resource_block.is_some());
18655            runner.merge().await.unwrap();
18656            assert!(runner.state.merge.is_none());
18657            drop(guard);
18658            runner.state = RunState::load(&runner.state.id).unwrap();
18659            // The driver clears the previous pass's resource stop on resume.
18660            runner.state.resource_block = None;
18661            runner.review_loop().await.unwrap();
18662            runner.gate().await.unwrap();
18663            assert!(runner.state.gate_allows_progress());
18664            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18665            assert_eq!(runner.state.gate_fixes.len(), 1);
18666        }
18667    }
18668
18669    #[cfg(unix)]
18670    #[tokio::test]
18671    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18672        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18673        runner.sync_to_base().await.unwrap();
18674        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18675        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18676        // Verification passes before pre_gate and fails on the mechanically changed head.
18677        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18678        runner.review_loop().await.unwrap();
18679        runner.gate().await.unwrap();
18680        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18681        assert_ne!(before, after);
18682        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18683        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18684        assert!(!runner.state.gate_allows_progress());
18685        runner.merge().await.unwrap();
18686        assert!(runner.state.merge.is_none());
18687        assert!(
18688            runner
18689                .state
18690                .verification_history
18691                .iter()
18692                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18693        );
18694
18695        // An incomplete review is still an independent hold on a changed head.
18696        runner.state.reviews[0].clean = false;
18697        runner.state.reviews[0].answered = 0;
18698        runner.state.config.verify.e2e = vec!["true".into()];
18699        runner.review_loop().await.unwrap();
18700        runner.gate().await.unwrap();
18701        runner.merge().await.unwrap();
18702        assert_eq!(runner.state.status, RunStatus::Blocked);
18703        assert!(runner.state.merge.is_none());
18704        assert!(
18705            runner
18706                .state
18707                .verification_blocking_cause()
18708                .unwrap()
18709                .contains("review blocked")
18710        );
18711    }
18712
18713    #[cfg(unix)]
18714    #[tokio::test]
18715    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18716        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18717        runner.state.gate_head = None;
18718        runner.state.config.verify.gate =
18719            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18720        runner.review_loop().await.unwrap();
18721        runner.gate().await.unwrap();
18722        assert_ne!(
18723            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18724            old
18725        );
18726        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18727        assert!(!runner.state.gate_allows_progress());
18728        assert_eq!(runner.state.status, RunStatus::Blocked);
18729        runner.merge().await.unwrap();
18730        assert!(runner.state.merge.is_none());
18731    }
18732
18733    #[cfg(unix)]
18734    #[tokio::test]
18735    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18736        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18737        let mut old_review = review_round(true, 0, 1, 1, false, true);
18738        old_review.head = head.clone();
18739        old_review.verified_head = Some(head.clone());
18740        runner.state.reviews = vec![old_review];
18741        runner.state.gate = vec![CommandOutcome {
18742            command: "old gate".into(),
18743            code: None,
18744            output_tail: "timed out after 2400s".into(),
18745            duration_ms: 2_400_000,
18746            resource_blocked: false,
18747        }];
18748        runner.state.gate_ran = true;
18749        runner.state.gate_head = Some(head.clone());
18750        runner.state.land_approval = Some(crate::run::LandApproval {
18751            question: "old-head-approval".into(),
18752            head: head.clone(),
18753        });
18754        runner.state.authorize_base_sync(&task).unwrap();
18755        runner.sync_to_base().await.unwrap();
18756        assert!(!runner.state.base_sync_blocked());
18757        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18758        assert_ne!(head, fresh);
18759        assert_eq!(runner.state.reviews.len(), 1);
18760        assert!(!runner.state.gate_ran);
18761        let saved = &runner.state.base_sync_recovery.authorizations[0];
18762        assert_eq!(saved.reviews[0].head, head);
18763        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18764        assert!(saved.gate_ran);
18765        assert!(!saved.gate[0].ok());
18766        runner.review_loop().await.unwrap();
18767        runner.gate().await.unwrap();
18768        assert_eq!(runner.state.reviews[0].head, head);
18769        assert_eq!(
18770            runner.state.reviews[0].verified_head.as_deref(),
18771            Some(head.as_str())
18772        );
18773        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18774        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18775        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18776        assert!(runner.state.gate_ran);
18777        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18778        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18779        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18780        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18781        assert_eq!(
18782            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18783            "retained change"
18784        );
18785    }
18786
18787    #[cfg(unix)]
18788    #[tokio::test]
18789    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18790        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18791        let queue = crate::queue::Queue::open();
18792        let mut owner = queue.get(&task).unwrap();
18793        owner.release_operator("test-resume", None);
18794        queue.put(&mut owner).unwrap();
18795        let generation = owner.resumption.as_ref().unwrap().generation;
18796        runner.state.task_instruction = Some(owner.instruction.clone());
18797        runner.state.driver_exited = true;
18798        runner.state.save().unwrap();
18799        let home = crate::run::home();
18800        let _claim = queue.claim(&task).unwrap();
18801        owner.resumption.as_mut().unwrap().at =
18802            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18803        queue.put(&mut owner).unwrap();
18804        assert!(
18805            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18806                .await
18807                .is_err(),
18808            "an approval from before the stop grants nothing"
18809        );
18810        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18811        queue.put(&mut owner).unwrap();
18812        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18813            .await
18814            .unwrap();
18815        assert!(
18816            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18817                .await
18818                .is_err()
18819        );
18820        runner.state = RunState::load(&runner.state.id).unwrap();
18821        assert_eq!(
18822            runner.state.base_sync_recovery.authorizations[0].generation,
18823            Some(generation)
18824        );
18825        runner.sync_to_base().await.unwrap();
18826        let repo = runner.state.repo.clone();
18827        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18828        git::git(&repo, &["checkout", "main"]).await.unwrap();
18829        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18830        git::git(&repo, &["add", "."]).await.unwrap();
18831        git::git(&repo, &["commit", "-m", "move base again"])
18832            .await
18833            .unwrap();
18834        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18835        git::git(&repo, &["checkout", "fix/retained"])
18836            .await
18837            .unwrap();
18838        runner.sync_to_base().await.unwrap();
18839        assert!(!runner.state.base_sync_blocked());
18840        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18841        assert_eq!(runner.state.base_episode_used(), 2);
18842        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18843        assert_ne!(fresh, head);
18844        assert!(!runner.state.base_recovery_pending());
18845        // Even replaying the generation with a later timestamp grants nothing.
18846        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18847        queue.put(&mut owner).unwrap();
18848        assert!(
18849            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18850                .await
18851                .is_err()
18852        );
18853        runner.state = RunState::load(&runner.state.id).unwrap();
18854        runner.sync_to_base().await.unwrap();
18855        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18856        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18857        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18858    }
18859
18860    #[cfg(unix)]
18861    #[tokio::test]
18862    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18863        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18864        runner.state.base_sync = None;
18865        runner.state.status = RunStatus::Blocked;
18866        let mut second = runner.state.candidates[0].clone();
18867        second.label = 'B';
18868        runner.state.candidates.push(second);
18869        assert!(runner.state.votes.is_empty());
18870        assert!(runner.state.deliberation.is_empty());
18871        runner.deliberate().await.unwrap();
18872        runner.vote().await.unwrap();
18873        assert_eq!(runner.state.status, RunStatus::Blocked);
18874        assert!(runner.state.votes.is_empty());
18875        assert!(runner.state.deliberation.is_empty());
18876    }
18877
18878    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18879    /// reads nothing else off it.
18880    fn runner_at(status: RunStatus) -> Runner {
18881        let mut state = RunState::new(
18882            PathBuf::from("/nonexistent/repo"),
18883            "main".to_owned(),
18884            "deadbeef".to_owned(),
18885            "task".to_owned(),
18886            Config::default(),
18887        );
18888        // These fixtures test execution flow, independent of host disk capacity.
18889        state.config.disk.min_free_bytes = 0;
18890        // These fixtures share one home and save concurrently. Short clock
18891        // IDs can collide, making an ownership test read another test's run.
18892        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18893        state.id = format!(
18894            "20000101-000000-{:04x}",
18895            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18896        );
18897        state.status = status;
18898        Runner {
18899            state,
18900            roles: ResolvedRoles {
18901                implementers: Vec::new(),
18902                judges: Vec::new(),
18903                reviewers: Vec::new(),
18904                fixer: None,
18905                conductor: conductor(),
18906                implementer_roster: Vec::new(),
18907                judge_roster: Vec::new(),
18908                reviewer_roster: Vec::new(),
18909            },
18910            sem: Arc::new(Semaphore::new(1)),
18911            pause: Pause::new(),
18912            interrupt: Pause::new(),
18913        }
18914    }
18915
18916    #[tokio::test]
18917    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18918        crate::run::pin_test_home();
18919        let temp = tempfile::tempdir().unwrap();
18920        let root = temp.path().canonicalize().unwrap();
18921        let repo = root.join("repo");
18922        std::fs::create_dir(&repo).unwrap();
18923        init_repo(&repo);
18924        git::git(&repo, &["config", "core.autocrlf", "false"])
18925            .await
18926            .unwrap();
18927        let mut runner = runner_at(RunStatus::Blocked);
18928        runner.state.repo = repo.clone();
18929        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18930        runner.state.config.agents = vec![conductor()];
18931        runner.state.config.graph.selected_implementer = Some("conductor".into());
18932        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18933        let (shell, command, setup_output) = if cfg!(windows) {
18934            (
18935                vec!["cmd".into(), "/C".into()],
18936                "echo ready>seat-ready",
18937                "ready\r\n",
18938            )
18939        } else {
18940            (
18941                vec!["sh".into(), "-c".into()],
18942                "printf ready > seat-ready",
18943                "ready",
18944            )
18945        };
18946        runner.state.config.verify.shell = Some(shell);
18947        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18948            run: Some(command.into()),
18949            ..Default::default()
18950        }];
18951        runner.state.save().unwrap();
18952        let path = runner.state.worktree_root().join("judge-1");
18953        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18954            .await
18955            .unwrap();
18956        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18957        git::git(&path, &["add", "."]).await.unwrap();
18958        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18959        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18960            .await
18961            .unwrap();
18962        git::git(&path, &["checkout", "--detach", &head])
18963            .await
18964            .unwrap();
18965        let evidence = crate::fold_evidence::snapshot(
18966            &runner.state,
18967            &crate::run::home().canonicalize().unwrap(),
18968            &head,
18969            64 * 1024 * 1024,
18970        )
18971        .await
18972        .unwrap();
18973        let mut contents = BTreeMap::new();
18974        contents.insert(
18975            repo.join("logs/HEAD"),
18976            format!("{head} {head} fixture\n").into_bytes(),
18977        );
18978        let record = serde_json::json!({
18979            "version":1, "run":runner.state.id, "source":path, "head":head,
18980            "evidence":evidence, "admin":{}, "admin_contents":contents,
18981            "history":{head.clone():evidence}, "phase":"evicted"
18982        });
18983        std::fs::write(
18984            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18985            serde_json::to_vec(&record).unwrap(),
18986        )
18987        .unwrap();
18988        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18989        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18990            .await
18991            .unwrap();
18992        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18993        assert!(
18994            !git::git_raw(&repo, &["cat-file", "-e", &head])
18995                .await
18996                .unwrap()
18997                .ok()
18998        );
18999        let mut resumed = Runner::resume(&runner.state.id).unwrap();
19000        let error = resumed.execute().await.unwrap_err();
19001        assert!(
19002            error
19003                .to_string()
19004                .contains("selected implementation is held"),
19005            "{error:#}"
19006        );
19007        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
19008        assert_eq!(
19009            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
19010            setup_output
19011        );
19012        assert_eq!(
19013            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
19014            "unrelated seat"
19015        );
19016        let error = resumed.execute().await.unwrap_err();
19017        assert!(
19018            error
19019                .to_string()
19020                .contains("selected implementation is held"),
19021            "{error:#}"
19022        );
19023        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
19024        git::git(&path, &["add", "."]).await.unwrap();
19025        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
19026        let next = git::git(
19027            &path,
19028            &[
19029                "commit-tree",
19030                &next_tree,
19031                "-p",
19032                &head,
19033                "-m",
19034                "next review checkpoint",
19035            ],
19036        )
19037        .await
19038        .unwrap();
19039        git::git(&path, &["checkout", "--detach", &next])
19040            .await
19041            .unwrap();
19042        resumed.state.reviews.push(
19043            serde_json::from_value(serde_json::json!({
19044                "round":1, "head":next, "reviews":[]
19045            }))
19046            .unwrap(),
19047        );
19048        resumed.state.save().unwrap();
19049        let error = resumed.execute().await.unwrap_err();
19050        assert!(
19051            error
19052                .to_string()
19053                .contains("selected implementation is held"),
19054            "{error:#}"
19055        );
19056        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
19057    }
19058
19059    #[tokio::test]
19060    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
19061        crate::run::pin_test_home();
19062        let mut runner = runner_at(RunStatus::Gating);
19063        runner.state.save().unwrap();
19064        let id = runner.state.id.clone();
19065        let passes = runner.state.passes.len();
19066        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19067        let mut claiming = Box::pin(runner.claim_driver());
19068        std::future::poll_fn(|cx| {
19069            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
19070            std::task::Poll::Ready(())
19071        })
19072        .await;
19073        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
19074        drop(owner);
19075        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
19076            .await
19077            .unwrap()
19078            .unwrap();
19079        assert_eq!(execution.runner.state.passes.len(), passes + 1);
19080        assert!(execution.runner.state.resource_block.is_none());
19081        assert_eq!(
19082            execution.runner.state.liveness(false),
19083            crate::run::Liveness::Live
19084        );
19085        drop(execution);
19086        assert!(RunState::load(&id).unwrap().driver_exited);
19087    }
19088
19089    #[tokio::test]
19090    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
19091        crate::run::pin_test_home();
19092        let mut runner = runner_at(RunStatus::Gating);
19093        runner.state.save().unwrap();
19094        // run.json can commit, but the subsequent lease metadata write fails.
19095        // Cleanup must run after releasing the short ownership lock.
19096        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
19097        assert!(runner.claim_driver().await.is_err());
19098        assert!(runner.state.driver_exited);
19099        assert_eq!(runner.state.driver_pid, None);
19100        let disk = RunState::load(&runner.state.id).unwrap();
19101        assert!(disk.driver_exited);
19102        assert_eq!(disk.driver_pid, None);
19103        assert_eq!(disk.status, RunStatus::Gating);
19104    }
19105
19106    #[tokio::test]
19107    async fn an_early_graph_error_releases_its_driver() {
19108        crate::run::pin_test_home();
19109        let mut runner = runner_at(RunStatus::Blocked);
19110        runner.state.config.graph.selected_implementer = Some("selected".into());
19111        runner.state.save().unwrap();
19112        assert!(runner.execute().await.is_err());
19113        let disk = RunState::load(&runner.state.id).unwrap();
19114        assert!(disk.driver_exited);
19115        assert_eq!(disk.driver_pid, None);
19116        assert_eq!(disk.status, RunStatus::Blocked);
19117    }
19118
19119    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
19120        crate::run::MergeRepair {
19121            question: "repair-question".into(),
19122            head: "old-head".into(),
19123            pr: "https://github.com/o/r/pull/42".into(),
19124            reviews: vec![review_round(false, 1, 1, 1, false, true)],
19125            gate: vec![outcome(Some(0), false)],
19126            gate_head: Some("old-head".into()),
19127            gate_fixes: Vec::new(),
19128            pre_gate: Vec::new(),
19129            phase,
19130            failure: None,
19131        }
19132    }
19133
19134    #[tokio::test]
19135    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
19136        crate::run::pin_test_home();
19137        let mut runner = runner_at(RunStatus::Landing);
19138        runner.state.merge_repairs.push(merge_repair_request(
19139            crate::run::MergeRepairPhase::Requested,
19140        ));
19141        runner.state.save().unwrap();
19142        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
19143        assert!(
19144            runner
19145                .execute()
19146                .await
19147                .unwrap_err()
19148                .to_string()
19149                .contains("already running")
19150        );
19151        assert_eq!(
19152            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19153            crate::run::MergeRepairPhase::Requested
19154        );
19155        drop(claim);
19156        let mut disk = runner.state.clone();
19157        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
19158        disk.save().unwrap();
19159        assert!(
19160            runner
19161                .execute()
19162                .await
19163                .unwrap_err()
19164                .to_string()
19165                .contains("decision changed")
19166        );
19167        assert_eq!(
19168            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19169            crate::run::MergeRepairPhase::Complete
19170        );
19171    }
19172
19173    #[test]
19174    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
19175        crate::run::pin_test_home();
19176        let mut runner = runner_at(RunStatus::Landing);
19177        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
19178        runner.state.reviews = request.reviews.clone();
19179        runner.state.gate = request.gate.clone();
19180        runner.state.gate_ran = true;
19181        runner.state.land_approval = Some(crate::run::LandApproval {
19182            question: "old".into(),
19183            head: "old-head".into(),
19184        });
19185        runner.state.merge_repairs.push(request);
19186        runner.begin_merge_repair_review(0).unwrap();
19187        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
19188        assert!(!runner.state.gate_status().ok());
19189        assert!(runner.state.land_approval.is_none());
19190        let disk = RunState::load(&runner.state.id).unwrap();
19191        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
19192        assert!(disk.merge_repairs[0].gate[0].ok());
19193        assert_eq!(
19194            disk.merge_repairs[0].phase,
19195            crate::run::MergeRepairPhase::Reviewing
19196        );
19197        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
19198        assert!(disk.reviews.is_empty());
19199    }
19200
19201    #[tokio::test]
19202    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
19203        crate::run::pin_test_home();
19204        for phase in [
19205            crate::run::MergeRepairPhase::Fixing,
19206            crate::run::MergeRepairPhase::Requested,
19207        ] {
19208            let mut runner = runner_at(RunStatus::Landing);
19209            runner.state.config.graph.land_rounds = 0;
19210            runner.state.merge_repairs.push(merge_repair_request(phase));
19211            runner.state.save().unwrap();
19212            assert!(!runner.repair_merge_review().await.unwrap());
19213            assert_eq!(runner.state.status, RunStatus::Blocked);
19214            assert_eq!(
19215                runner.state.merge_repairs[0].phase,
19216                crate::run::MergeRepairPhase::Failed
19217            );
19218            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
19219            assert!(
19220                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
19221                    "interrupted"
19222                } else {
19223                    "budget exhausted"
19224                })
19225            );
19226            assert!(runner.state.merge.is_none());
19227        }
19228    }
19229
19230    /// `park_here` folding in the reason `Pause::park_because` recorded -
19231    /// this is what lets an operator reading a run's events tell an
19232    /// interrupt-driven park from an ordinary shutdown park.
19233    #[test]
19234    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
19235        crate::run::pin_test_home();
19236        let mut runner = runner_at(RunStatus::Implementing);
19237        let interrupt = Pause::new();
19238        runner.watch_interrupt(interrupt.clone());
19239
19240        interrupt.park_because("task a1b2 asked to run first");
19241
19242        assert!(runner.park_here().expect("park_here"));
19243        assert!(runner.state.parked);
19244        let last = runner.state.events.last().expect("a park event");
19245        assert_eq!(last.node, "park");
19246        assert!(
19247            last.message.contains("task a1b2 asked to run first"),
19248            "expected the interrupt reason in {:?}",
19249            last.message
19250        );
19251    }
19252
19253    /// The cause is stored on the pass when it parks, and the first cause a
19254    /// shared pause was given is the one that stays.
19255    #[test]
19256    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
19257        crate::run::pin_test_home();
19258        let mut runner = runner_at(RunStatus::Implementing);
19259        runner.state.begin_pass();
19260        let shutdown = Pause::new();
19261        runner.on_pause(shutdown.clone());
19262        let upgrade = ParkCause::Upgrade {
19263            from: "0.121.0".to_owned(),
19264            to: Some("0.121.1".to_owned()),
19265        };
19266        shutdown.park_for(upgrade.clone());
19267        shutdown.park_for(ParkCause::Operator);
19268
19269        assert!(runner.park_here().expect("park_here"));
19270        assert_eq!(
19271            runner.state.passes[0].end,
19272            Some(crate::run::PassEnd::Parked { cause: upgrade })
19273        );
19274        let last = runner.state.events.last().expect("a park event");
19275        assert!(last.message.contains("0.121.0"), "{}", last.message);
19276    }
19277
19278    #[tokio::test]
19279    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
19280        crate::run::pin_test_home();
19281        let mut runner = runner_at(RunStatus::Reviewing);
19282        runner.state.save().unwrap();
19283        let mut execution = runner.claim_driver().await.expect("first execution");
19284        assert_eq!(execution.runner.state.passes.len(), 1);
19285        execution
19286            .driver
19287            .finish(&mut execution.runner.state)
19288            .await
19289            .unwrap();
19290    }
19291
19292    #[tokio::test]
19293    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19294        crate::run::pin_test_home();
19295        let mut runner = runner_at(RunStatus::Reviewing);
19296        runner.state.config.agents = vec![conductor()];
19297        runner.state.config.graph.timeout_resource_wait = 0;
19298        runner.state.save().unwrap();
19299        let id = runner.state.id.clone();
19300        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19301        assert!(runner.claim_driver().await.is_err());
19302        drop(owner);
19303        drop(runner);
19304        let mut resumed = Runner::resume(&id).unwrap();
19305        assert_eq!(resumed.state.driver_claimed, Some(false));
19306        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19307        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19308        let mut execution = resumed
19309            .claim_driver()
19310            .await
19311            .expect("retry initial claim after reload");
19312        assert_eq!(execution.runner.state.driver_claimed, Some(true));
19313        assert_eq!(execution.runner.state.passes.len(), 1);
19314        execution
19315            .driver
19316            .finish(&mut execution.runner.state)
19317            .await
19318            .unwrap();
19319    }
19320
19321    #[tokio::test]
19322    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19323        crate::run::pin_test_home();
19324        let mut runner = runner_at(RunStatus::Reviewing);
19325        runner.state.config.agents = vec![conductor()];
19326        runner.state.save().unwrap();
19327        let id = runner.state.id.clone();
19328        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19329        let mut resumed = Runner::resume(&id).unwrap();
19330        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19331        assert!(resumed.claim_driver().await.is_err());
19332        assert!(RunState::load(&id).unwrap().passes.is_empty());
19333        drop(lease);
19334        let mut execution = resumed.claim_driver().await.unwrap();
19335        execution
19336            .driver
19337            .finish(&mut execution.runner.state)
19338            .await
19339            .unwrap();
19340    }
19341
19342    #[tokio::test]
19343    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19344        crate::run::pin_test_home();
19345        let mut runner = runner_at(RunStatus::Reviewing);
19346        let state = &mut runner.state;
19347        state.config.agents = vec![conductor()];
19348        state.driver_claimed = None;
19349        state.driver_pid = Some(std::process::id());
19350        state.driver_started_at = None;
19351        state.begin_pass();
19352        state.save().unwrap();
19353        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19354        std::fs::write(
19355            state.dir().join("driver.json"),
19356            serde_json::to_vec(&serde_json::json!({
19357                "pid": state.driver_pid, "started_at": null,
19358                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19359            }))
19360            .unwrap(),
19361        )
19362        .unwrap();
19363        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19364        drop(lease);
19365        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19366        let mut resumed = Runner::resume(&state.id).unwrap();
19367        assert!(!resumed.state.driver_exited);
19368        let mut execution = resumed
19369            .claim_driver()
19370            .await
19371            .expect("lease proves previous execution ended");
19372        assert_eq!(execution.runner.state.passes.len(), 2);
19373        execution
19374            .driver
19375            .finish(&mut execution.runner.state)
19376            .await
19377            .unwrap();
19378    }
19379
19380    #[tokio::test]
19381    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19382        crate::run::pin_test_home();
19383        let mut runner = runner_at(RunStatus::Reviewing);
19384        let mut json = serde_json::to_value(&runner.state).unwrap();
19385        json.as_object_mut().unwrap().remove("driver_claimed");
19386        runner.state = serde_json::from_value(json).unwrap();
19387        assert_eq!(runner.state.driver_claimed, None);
19388        runner.state.save().unwrap();
19389        let mut execution = runner
19390            .claim_driver()
19391            .await
19392            .expect("the execution lease, not a missing marker, excludes a second driver");
19393        assert_eq!(execution.runner.state.passes.len(), 1);
19394        execution
19395            .driver
19396            .finish(&mut execution.runner.state)
19397            .await
19398            .unwrap();
19399    }
19400
19401    #[tokio::test]
19402    async fn driver_admission_refuses_a_proven_live_driver() {
19403        crate::run::pin_test_home();
19404        let mut runner = runner_at(RunStatus::Reviewing);
19405        let pid = std::process::id();
19406        runner.state.driver_pid = Some(pid);
19407        runner.state.driver_started_at = crate::proc::process_started_at(pid);
19408        assert!(runner.state.driver_started_at.is_some());
19409        runner.state.save().unwrap();
19410        let error = runner
19411            .claim_driver()
19412            .await
19413            .err()
19414            .expect("a live driver must refuse");
19415        assert!(error.to_string().contains("live driver"), "{error:#}");
19416        let saved = RunState::load(&runner.state.id).unwrap();
19417        assert_eq!(saved.driver_pid, Some(pid));
19418        assert_eq!(saved.status, RunStatus::Reviewing);
19419        assert!(saved.passes.is_empty());
19420    }
19421
19422    #[test]
19423    fn parking_resource_halt_persists_the_pass_once() {
19424        crate::run::pin_test_home();
19425        let mut runner = runner_at(RunStatus::Reviewing);
19426        runner.state.begin_pass();
19427        runner.state.resource_block = Some(crate::run::ResourceBlock {
19428            node: "fix".into(),
19429            kind: "shared build cache".into(),
19430            detail: "cancelled".into(),
19431            at: Timestamp::now(),
19432            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19433        });
19434        let cause = ParkCause::Upgrade {
19435            from: "old".into(),
19436            to: Some("new".into()),
19437        };
19438        runner.pause.park_for(cause.clone());
19439        // Both direct fixer exits and graph resource boundaries must complete
19440        // parking, rather than returning a stale Reviewing record.
19441        runner.stop_for_resource().unwrap();
19442        assert!(runner.resource_halt().unwrap());
19443        assert!(runner.park_here().unwrap());
19444        let saved = RunState::load(&runner.state.id).unwrap();
19445        assert!(saved.parked);
19446        assert_eq!(saved.status, RunStatus::Reviewing);
19447        assert!(saved.passes[0].ended_at.is_some());
19448        assert_eq!(
19449            saved.passes[0].end,
19450            Some(crate::run::PassEnd::Parked {
19451                cause: cause.clone()
19452            })
19453        );
19454        assert_eq!(
19455            saved.resource_block.unwrap().cause,
19456            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19457        );
19458        assert_eq!(
19459            saved
19460                .events
19461                .iter()
19462                .filter(|event| event.node == "park")
19463                .count(),
19464            1
19465        );
19466    }
19467
19468    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
19469    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
19470    /// cleared) must not make a *different* run - one only watching its own,
19471    /// unshared interrupt `Pause` - see itself as parked. If a future change
19472    /// ever collapsed these back into one handle, the interrupt scheduler
19473    /// would park every run for the rest of the daemon's life, not just the
19474    /// one it meant to interrupt.
19475    #[test]
19476    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19477        crate::run::pin_test_home();
19478        let mut runner = runner_at(RunStatus::Implementing);
19479        let shutdown = Pause::new();
19480        runner.on_pause(shutdown.clone());
19481        let interrupt = Pause::new();
19482        runner.watch_interrupt(interrupt.clone());
19483
19484        // Nobody has asked for anything yet.
19485        assert!(!runner.park_here().expect("park_here"));
19486        assert!(!runner.state.parked);
19487
19488        // Only the interrupt handle fires; the shutdown handle stays clear.
19489        interrupt.park_because("test");
19490        assert!(!shutdown.parked());
19491        assert!(runner.park_here().expect("park_here"));
19492    }
19493
19494    /// The property every prior attempt at this feature failed to pin down:
19495    /// asking a run to park while one of its nodes has a real, in-flight
19496    /// async operation running (an agent call, in production) must not cut
19497    /// that operation short. `park_here` is only ever consulted *between*
19498    /// `execute`'s node calls - see its own doc - so nothing inside a node
19499    /// can observe a park request until the node itself returns. This proves
19500    /// that structurally, with real `tokio` concurrency and a channel
19501    /// handshake (never a sleep, which would only prove "usually", not
19502    /// "cannot"): the "node" below reports that it has genuinely started,
19503    /// and only then is the park requested; the node still has to be told to
19504    /// finish before `park_here` is ever called, exactly mirroring every
19505    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
19506    /// in `execute`.
19507    #[tokio::test]
19508    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19509        crate::run::pin_test_home();
19510        let mut runner = runner_at(RunStatus::Implementing);
19511        let interrupt = Pause::new();
19512        runner.watch_interrupt(interrupt.clone());
19513
19514        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19515        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19516
19517        // Stands in for one node's in-flight agent call: it proves it has
19518        // genuinely started, then blocks - exactly as a spawned CLI process
19519        // does - until told to finish.
19520        let node = async move {
19521            started_tx.send(()).expect("send started");
19522            finish_rx.await.expect("recv finish");
19523            "node finished"
19524        };
19525
19526        let interrupter = async move {
19527            started_rx.await.expect("recv started");
19528            // The call is now genuinely in flight. Ask it to park.
19529            interrupt.park_because("higher-priority task waiting");
19530            // Nothing the node does can observe this yet - there is no
19531            // check inside it, by construction - so let the executor run
19532            // anything pending and then let the node finish on its own.
19533            tokio::task::yield_now().await;
19534            finish_tx.send(()).expect("send finish");
19535        };
19536
19537        let (node_result, ()) = tokio::join!(node, interrupter);
19538        assert_eq!(
19539            node_result, "node finished",
19540            "the in-flight call ran to completion"
19541        );
19542
19543        // Only now, at the boundary the real `execute` would check right
19544        // after this node, does the park take effect.
19545        assert!(runner.park_here().expect("park_here"));
19546        assert!(runner.state.parked);
19547    }
19548
19549    /// A run parked mid-competition carries every field it had accumulated
19550    /// through the exact same disk round-trip an ordinary resume uses -
19551    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
19552    /// Nothing about parking for an interrupt is a special case of that path;
19553    /// this is what proves it rather than assuming it.
19554    #[test]
19555    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19556        crate::run::pin_test_home();
19557        let mut runner = runner_at(RunStatus::Judging);
19558        // `Runner::resume` re-resolves roles from the saved config, which
19559        // refuses an empty roster - give it the same minimal one `conductor`
19560        // itself uses.
19561        runner.state.config.agents = vec![conductor()];
19562        runner.state.candidates = vec![Candidate {
19563            index: 0,
19564            label: 'A',
19565            agent: "alpha".to_owned(),
19566            branch: "magi/x/A".to_owned(),
19567            worktree: PathBuf::from("/nonexistent/worktree"),
19568            summary: "did the thing".to_owned(),
19569            stat: "1 file changed".to_owned(),
19570            files: 1,
19571            commits: 1,
19572            empty: false,
19573            failed: None,
19574            verified_noop: None,
19575            duration_ms: 1234,
19576            folded: false,
19577        }];
19578        let run_id = runner.state.id.clone();
19579
19580        let interrupt = Pause::new();
19581        runner.watch_interrupt(interrupt.clone());
19582        interrupt.park_because("task c3d4 asked to run first");
19583        assert!(runner.park_here().expect("park_here"));
19584
19585        let resumed = Runner::resume(&run_id).expect("resume");
19586        assert_eq!(resumed.state.candidates.len(), 1);
19587        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19588        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19589        assert_eq!(resumed.state.status, runner.state.status);
19590        assert!(
19591            resumed.state.parked,
19592            "still parked until `execute` actually walks the graph again"
19593        );
19594        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19595    }
19596
19597    /// A fresh open question on `run`, stored and handed back for assertions.
19598    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19599        let mut q = ask::Question::new(
19600            run.to_owned(),
19601            "implement".to_owned(),
19602            "impl-A".to_owned(),
19603            "Which storage backend should the cache use?".to_owned(),
19604            String::new(),
19605            vec!["SQLite".to_owned(), "Redis".to_owned()],
19606        );
19607        store.put(&mut q).unwrap();
19608        q
19609    }
19610
19611    #[test]
19612    fn a_failed_runs_open_question_is_abandoned() {
19613        ask_test_home();
19614        let store = ask::Questions::open();
19615        let mut runner = runner_at(RunStatus::Failed);
19616        let run = runner.state.id.clone();
19617        let q = ask_open_question(&store, &run);
19618
19619        runner.settle_questions();
19620
19621        let back = store.get(&q.id).unwrap();
19622        assert!(
19623            !back.status.open(),
19624            "the seat that asked died with the run; nobody is left to read an answer"
19625        );
19626        assert!(
19627            back.detail.contains(&run) && back.detail.contains("failed"),
19628            "the reason names what the run became, not just that it is gone: {}",
19629            back.detail
19630        );
19631    }
19632
19633    #[test]
19634    fn a_merged_runs_open_question_is_abandoned_too() {
19635        ask_test_home();
19636        let store = ask::Questions::open();
19637        // A run that finishes cleanly still leaves nobody to read an answer -
19638        // this is not only a failure-path cleanup.
19639        for status in [RunStatus::Merged, RunStatus::Ready] {
19640            let mut runner = runner_at(status);
19641            let run = runner.state.id.clone();
19642            let q = ask_open_question(&store, &run);
19643
19644            runner.settle_questions();
19645
19646            let back = store.get(&q.id).unwrap();
19647            assert!(
19648                !back.status.open(),
19649                "{status:?} run's question must not outlive the run"
19650            );
19651        }
19652    }
19653
19654    #[test]
19655    fn a_still_resumable_runs_open_question_is_left_alone() {
19656        ask_test_home();
19657        let store = ask::Questions::open();
19658        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19659        // review round and the seat sessions are all still on disk — so a
19660        // question asked mid-round may yet get a real answer from a real
19661        // resume. Sweeping it here would be exactly the failure mode this
19662        // whole feature exists to avoid on the other side.
19663        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19664            let mut runner = runner_at(status);
19665            let run = runner.state.id.clone();
19666            let q = ask_open_question(&store, &run);
19667
19668            runner.settle_questions();
19669
19670            let back = store.get(&q.id).unwrap();
19671            assert!(
19672                back.status.open(),
19673                "{status:?} is still alive; the question must still be waiting"
19674            );
19675        }
19676    }
19677
19678    #[test]
19679    fn settle_questions_never_touches_an_already_answered_question() {
19680        ask_test_home();
19681        let store = ask::Questions::open();
19682        let mut runner = runner_at(RunStatus::Failed);
19683        let run = runner.state.id.clone();
19684        let mut q = ask_open_question(&store, &run);
19685        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19686            .unwrap();
19687        store.put(&mut q).unwrap();
19688
19689        // Called twice, the way a crash-recovered daemon reclaim and the
19690        // graph's own cleanup both can for the same run — `abandon_for_run`
19691        // only ever touches what is still open, so this must be inert both
19692        // times, not merely the second.
19693        runner.settle_questions();
19694        runner.settle_questions();
19695
19696        let back = store.get(&q.id).unwrap();
19697        assert_eq!(
19698            back.status,
19699            ask::QuestionStatus::Answered,
19700            "a real answer is a decision on record, never overwritten by a sweep"
19701        );
19702    }
19703
19704    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19705    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19706    /// without merging, whose winner is still the operator's answer to read.
19707    /// Nothing previously called `fold_run` itself with a real `tally`, so
19708    /// this is the first test to pin down the one distinction the whole
19709    /// automatic-fold feature depends on: the winner's worktree and branch
19710    /// must survive, everything else sharing the run's worktree bay - a
19711    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19712    /// second sweep treats every non-winner directory under the bay alike -
19713    /// must not.
19714    #[tokio::test]
19715    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19716        crate::run::pin_test_home();
19717        let tmp = tempfile::tempdir().expect("tempdir");
19718        let repo = tmp.path().join("repo");
19719        std::fs::create_dir_all(&repo).unwrap();
19720        init_repo(&repo);
19721
19722        let mut config = Config::default();
19723        config.graph.worktree_root = Some(tmp.path().join("wt"));
19724
19725        let mut state = RunState::new(
19726            repo.clone(),
19727            "main".to_owned(),
19728            "deadbeef".to_owned(),
19729            "task".to_owned(),
19730            config,
19731        );
19732        let root = state.worktree_root();
19733        let wt_a = root.join("cand-A");
19734        let wt_b = root.join("cand-B");
19735        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19736            .await
19737            .expect("worktree A");
19738        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19739            .await
19740            .expect("worktree B");
19741
19742        state.candidates = vec![
19743            Candidate {
19744                index: 0,
19745                label: 'A',
19746                agent: "alpha".to_owned(),
19747                branch: "magi/x/A".to_owned(),
19748                worktree: wt_a.clone(),
19749                summary: String::new(),
19750                stat: String::new(),
19751                files: 0,
19752                commits: 0,
19753                empty: false,
19754                failed: None,
19755                verified_noop: None,
19756                duration_ms: 0,
19757                folded: false,
19758            },
19759            Candidate {
19760                index: 1,
19761                label: 'B',
19762                agent: "beta".to_owned(),
19763                branch: "magi/x/B".to_owned(),
19764                worktree: wt_b.clone(),
19765                summary: String::new(),
19766                stat: String::new(),
19767                files: 0,
19768                commits: 0,
19769                empty: false,
19770                failed: None,
19771                verified_noop: None,
19772                duration_ms: 0,
19773                folded: false,
19774            },
19775        ];
19776        state.tally = Some(Tally {
19777            first_choice: BTreeMap::from([('A', 1)]),
19778            borda: BTreeMap::new(),
19779            winner: 'A',
19780            rankings: 1,
19781            unanimous_initial: true,
19782            deliberated: false,
19783            changed_votes: 0,
19784            unanimous_final: true,
19785            tie_break: None,
19786            judges: 1,
19787            present: 1,
19788            quorum: 1,
19789            met_quorum: true,
19790            uncontested: None,
19791        });
19792        state.status = RunStatus::Ready;
19793
19794        fold_run(&mut state, false, &crate::run::home())
19795            .await
19796            .expect("fold_run");
19797
19798        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19799        assert!(
19800            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19801            "the unmerged winner's branch survives"
19802        );
19803        assert!(
19804            !state.candidates[0].folded,
19805            "the winner is not marked folded"
19806        );
19807
19808        assert!(!wt_b.exists(), "the loser's worktree is removed");
19809        assert!(
19810            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19811            "the loser's branch is removed"
19812        );
19813        assert!(state.candidates[1].folded, "the loser is marked folded");
19814    }
19815
19816    #[tokio::test]
19817    #[cfg(unix)]
19818    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19819        use std::os::fd::AsRawFd as _;
19820        use std::time::Duration;
19821
19822        let tmp = tempfile::tempdir().unwrap();
19823        let repo = tmp.path().join("repo");
19824        std::fs::create_dir_all(&repo).unwrap();
19825        init_repo(&repo);
19826        let common = repo.join(".git");
19827        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19828        let home = tmp.path().join("home");
19829        let mut config = Config::default();
19830        config.coordination = crate::config::test_coordination();
19831        config.graph.worktree_root = Some(tmp.path().join("wt"));
19832        config.graph.timeout_resource_wait = 1;
19833        let mut state = RunState::new(
19834            repo.clone(),
19835            "main".into(),
19836            "deadbeef".into(),
19837            "task".into(),
19838            config,
19839        );
19840        let root = state.worktree_root();
19841        let candidate = root.join("cand-A");
19842        let reviewer = root.join("review-1");
19843        let branch = state.branch_for('A');
19844        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19845            .await
19846            .unwrap();
19847        git::worktree_add_detached(&repo, &reviewer, "main")
19848            .await
19849            .unwrap();
19850        git::acquire_worktree_config(&repo).await.unwrap();
19851        state.enabled_worktree_config = true;
19852        state.status = RunStatus::Ready;
19853        state.candidates.push(Candidate {
19854            index: 0,
19855            label: 'A',
19856            agent: "implement".into(),
19857            branch: branch.clone(),
19858            worktree: candidate.clone(),
19859            summary: String::new(),
19860            stat: String::new(),
19861            files: 0,
19862            commits: 0,
19863            empty: false,
19864            failed: None,
19865            verified_noop: None,
19866            duration_ms: 0,
19867            folded: false,
19868        });
19869        state.save_under(&home).unwrap();
19870        let config_before = std::fs::read(common.join("config")).unwrap();
19871        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19872            .await
19873            .unwrap();
19874        // A separate OS handle bypasses the launcher's local gate, exactly as
19875        // an external updater does, and must still prevent all folding work.
19876        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19877        assert_eq!(
19878            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19879            0
19880        );
19881        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19882        assert!(
19883            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19884                .await
19885                .is_err()
19886        );
19887        assert!(candidate.exists());
19888        assert!(reviewer.exists());
19889        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19890        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19891        assert_eq!(
19892            git::git(&repo, &["worktree", "list", "--porcelain"])
19893                .await
19894                .unwrap(),
19895            worktrees_before
19896        );
19897        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19898            .await
19899            .unwrap()
19900            .unwrap_err();
19901        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19902        drop(fold);
19903        assert!(candidate.exists() && reviewer.exists());
19904        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19905        assert_eq!(
19906            state.resource_block.as_ref().unwrap().kind,
19907            "repository coordination"
19908        );
19909        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19910        drop(external);
19911        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19912        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19913            .await
19914            .unwrap()
19915            .unwrap();
19916        drop(fold);
19917        assert!(removed.contains(&branch));
19918        assert!(!root.exists());
19919        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19920        assert!(state.candidates[0].folded);
19921        assert!(!state.enabled_worktree_config);
19922        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19923        let enabled = git::git_raw(
19924            &repo,
19925            &["config", "--local", "--get", "extensions.worktreeConfig"],
19926        )
19927        .await
19928        .unwrap();
19929        assert!(!enabled.ok(), "the last config reference was released");
19930    }
19931
19932    /// A branch handed to a later run is that run's (and its pull request's):
19933    /// folding the run that released it must not delete it.
19934    #[tokio::test]
19935    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19936        let tmp = tempfile::tempdir().expect("tempdir");
19937        let repo = tmp.path().join("repo");
19938        std::fs::create_dir_all(&repo).unwrap();
19939        init_repo(&repo);
19940        let home = tmp.path().join("home");
19941
19942        let mut config = Config::default();
19943        config.graph.worktree_root = Some(tmp.path().join("wt"));
19944        let mut state = RunState::new(
19945            repo.clone(),
19946            "main".to_owned(),
19947            "deadbeef".to_owned(),
19948            "task".to_owned(),
19949            config,
19950        );
19951        // The worktree is already gone (released); the branch survives.
19952        git::git(&repo, &["branch", "magi/x/A", "main"])
19953            .await
19954            .expect("branch");
19955        state.candidates = vec![Candidate {
19956            index: 0,
19957            label: 'A',
19958            agent: "alpha".to_owned(),
19959            branch: "magi/x/A".to_owned(),
19960            worktree: state.worktree_root().join("cand-A"),
19961            summary: String::new(),
19962            stat: String::new(),
19963            files: 0,
19964            commits: 0,
19965            empty: false,
19966            failed: None,
19967            verified_noop: None,
19968            duration_ms: 0,
19969            folded: true,
19970        }];
19971        state.released_to = Some("20260901-000000-new1".to_owned());
19972        state.released_branches = vec!["magi/x/A".to_owned()];
19973
19974        fold_run(&mut state, true, &home).await.expect("fold_run");
19975        state.status = RunStatus::Merged;
19976        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19977        state.merge = Some(crate::run::MergeOutcome {
19978            mode: MergeMode::Pr,
19979            ok: true,
19980            detail: "confirmed merged".into(),
19981            empty: false,
19982        });
19983        fold_run(&mut state, true, &home)
19984            .await
19985            .expect("merged fold");
19986
19987        assert!(
19988            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19989            "the handed-over branch survives a fold"
19990        );
19991    }
19992
19993    /// A winner with nothing ahead of the base is caught before `gh` is ever
19994    /// asked for a pull request, and the message carries what the task's
19995    /// references resolved to.
19996    #[tokio::test]
19997    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19998        let tmp = tempfile::tempdir().expect("tempdir");
19999        let repo = tmp.path().join("repo");
20000        std::fs::create_dir_all(&repo).unwrap();
20001        init_repo(&repo);
20002        let run = |args: &[&str]| {
20003            let out = std::process::Command::new("git")
20004                .quiet()
20005                .args(args)
20006                .current_dir(&repo)
20007                .output()
20008                .expect("spawn git");
20009            assert!(out.status.success(), "git {args:?}");
20010        };
20011        run(&["branch", "magi/x/A"]);
20012        run(&["checkout", "-q", "-b", "magi/x/B"]);
20013        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
20014        run(&["add", "-A"]);
20015        run(&["commit", "-q", "-m", "work"]);
20016        run(&["checkout", "-q", "main"]);
20017
20018        // A pull request is compared against the remote's base, so the
20019        // fixture needs one. Before it exists nothing can be read, and the
20020        // answer must be "not empty".
20021        let probe = RunState::new(
20022            repo.clone(),
20023            "main".to_owned(),
20024            "deadbeef".to_owned(),
20025            "task".to_owned(),
20026            Config::default(),
20027        );
20028        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
20029        let bare = tmp.path().join("origin.git");
20030        let out = std::process::Command::new("git")
20031            .quiet()
20032            .args(["init", "-q", "--bare"])
20033            .arg(&bare)
20034            .output()
20035            .expect("spawn git");
20036        assert!(out.status.success(), "git init --bare");
20037        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
20038        run(&["push", "-q", "origin", "main"]);
20039
20040        let mut state = RunState::new(
20041            repo.clone(),
20042            "main".to_owned(),
20043            "deadbeef".to_owned(),
20044            "task".to_owned(),
20045            Config::default(),
20046        );
20047        state.seeds = vec![refs::Seed {
20048            token: "magi/27b2/A".to_owned(),
20049            kind: refs::SeedKind::Unresolved,
20050            sha: String::new(),
20051            branch: true,
20052            detail: "no branch or commit named magi/27b2/A".to_owned(),
20053        }];
20054
20055        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
20056        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
20057        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
20058        let detail = empty_candidate_detail(&state, "main");
20059        assert!(detail.starts_with("empty candidate"), "{detail}");
20060        assert!(!detail.contains("magi/27b2/A"), "{detail}");
20061        assert!(detail.contains("--start-from"), "{detail}");
20062    }
20063
20064    /// `status == Ready` used to be read as "this is the harmless
20065    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
20066    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
20067    /// run's PR was closed without merging — and reentering `merge` with
20068    /// `mode` still `Pr` does not know the difference, so it pushed and
20069    /// opened a second pull request. `mode == Local` reproduces the same
20070    /// blind spot without a network call: reentry must not attempt another
20071    /// git merge once this node has already recorded an outcome.
20072    #[tokio::test]
20073    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
20074        let tmp = tempfile::tempdir().expect("tempdir");
20075        let repo = tmp.path().join("repo");
20076        std::fs::create_dir_all(&repo).unwrap();
20077        init_repo(&repo);
20078
20079        let mut config = Config::default();
20080        config.merge.mode = MergeMode::Local;
20081
20082        let mut state = RunState::new(
20083            repo.clone(),
20084            "main".to_owned(),
20085            "deadbeef".to_owned(),
20086            "task".to_owned(),
20087            config,
20088        );
20089        state.candidates = vec![Candidate {
20090            index: 0,
20091            label: 'A',
20092            agent: "alpha".to_owned(),
20093            branch: "does-not-exist".to_owned(),
20094            worktree: repo.clone(),
20095            summary: String::new(),
20096            stat: String::new(),
20097            files: 0,
20098            commits: 0,
20099            empty: false,
20100            failed: None,
20101            verified_noop: None,
20102            duration_ms: 0,
20103            folded: false,
20104        }];
20105        state.tally = Some(Tally {
20106            first_choice: BTreeMap::from([('A', 1)]),
20107            borda: BTreeMap::new(),
20108            winner: 'A',
20109            rankings: 1,
20110            unanimous_initial: true,
20111            deliberated: false,
20112            changed_votes: 0,
20113            unanimous_final: true,
20114            tie_break: None,
20115            judges: 0,
20116            present: 0,
20117            quorum: 0,
20118            met_quorum: true,
20119            uncontested: Some("only candidate A produced a change".to_owned()),
20120        });
20121        state.reviews = vec![ReviewRound {
20122            round: 1,
20123            head: "deadbeef".to_owned(),
20124            verified_head: None,
20125            verified_at: None,
20126            reviews: Vec::new(),
20127            e2e: Vec::new(),
20128            fix: None,
20129            blocking: 0,
20130            answered: 0,
20131            expected: 0,
20132            clean: true,
20133            verify_retried: false,
20134            e2e_deferred: false,
20135            e2e_defer_reason: None,
20136            progressed: false,
20137            vote_split: false,
20138            reconsideration: Vec::new(),
20139            verdict: None,
20140        }];
20141        state.gate = vec![CommandOutcome {
20142            command: "test".to_owned(),
20143            code: Some(0),
20144            output_tail: String::new(),
20145            duration_ms: 0,
20146            resource_blocked: false,
20147        }];
20148        state.gate_ran = true;
20149        // Reached its conclusion already — e.g. `land` closing the PR without
20150        // merging it, which (like the honest `MergeMode::None` path) leaves
20151        // `status` at `Ready`. The recorded outcome is what actually marks
20152        // this node done.
20153        state.status = RunStatus::Ready;
20154        state.merge = Some(MergeOutcome {
20155            mode: MergeMode::Local,
20156            ok: false,
20157            detail: "already concluded".to_owned(),
20158            empty: false,
20159        });
20160
20161        let mut runner = Runner {
20162            state,
20163            roles: ResolvedRoles {
20164                implementers: Vec::new(),
20165                judges: Vec::new(),
20166                reviewers: Vec::new(),
20167                fixer: None,
20168                conductor: conductor(),
20169                implementer_roster: Vec::new(),
20170                judge_roster: Vec::new(),
20171                reviewer_roster: Vec::new(),
20172            },
20173            sem: Arc::new(Semaphore::new(1)),
20174            pause: Pause::new(),
20175            interrupt: Pause::new(),
20176        };
20177
20178        runner.merge().await.expect("merge");
20179
20180        assert_eq!(
20181            runner.state.status,
20182            RunStatus::Ready,
20183            "a concluded run's status must not change on reentry"
20184        );
20185        assert_eq!(
20186            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20187            Some("already concluded"),
20188            "merge must not run again once the node already recorded an outcome"
20189        );
20190    }
20191
20192    /// `gate` leaves `state.gate_ran` false both before it has ever run and
20193    /// when its last attempt was resource-blocked (the shared build cache
20194    /// could not be acquired or confirmed fresh in time - see
20195    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
20196    /// `Vec` this also leaves behind used to read as "nothing failed" and let
20197    /// a run merge a tree the gate never actually checked - exactly the case
20198    /// a contended cache produces on every retry until it clears. `merge`
20199    /// must refuse until `gate` has actually recorded an attempt.
20200    #[tokio::test]
20201    async fn merge_refuses_a_gate_that_has_not_actually_run() {
20202        let tmp = tempfile::tempdir().expect("tempdir");
20203        let repo = tmp.path().join("repo");
20204        std::fs::create_dir_all(&repo).unwrap();
20205        init_repo(&repo);
20206
20207        let mut config = Config::default();
20208        config.merge.mode = MergeMode::Local;
20209
20210        let mut state = RunState::new(
20211            repo.clone(),
20212            "main".to_owned(),
20213            "deadbeef".to_owned(),
20214            "task".to_owned(),
20215            config,
20216        );
20217        state.candidates = vec![Candidate {
20218            index: 0,
20219            label: 'A',
20220            agent: "alpha".to_owned(),
20221            branch: "does-not-exist".to_owned(),
20222            worktree: repo.clone(),
20223            summary: String::new(),
20224            stat: String::new(),
20225            files: 0,
20226            commits: 0,
20227            empty: false,
20228            failed: None,
20229            verified_noop: None,
20230            duration_ms: 0,
20231            folded: false,
20232        }];
20233        state.tally = Some(Tally {
20234            first_choice: BTreeMap::from([('A', 1)]),
20235            borda: BTreeMap::new(),
20236            winner: 'A',
20237            rankings: 1,
20238            unanimous_initial: true,
20239            deliberated: false,
20240            changed_votes: 0,
20241            unanimous_final: true,
20242            tie_break: None,
20243            judges: 0,
20244            present: 0,
20245            quorum: 0,
20246            met_quorum: true,
20247            uncontested: Some("only candidate A produced a change".to_owned()),
20248        });
20249        state.reviews = vec![ReviewRound {
20250            round: 1,
20251            head: "deadbeef".to_owned(),
20252            verified_head: None,
20253            verified_at: None,
20254            reviews: Vec::new(),
20255            e2e: Vec::new(),
20256            fix: None,
20257            blocking: 0,
20258            answered: 0,
20259            expected: 0,
20260            clean: true,
20261            verify_retried: false,
20262            e2e_deferred: false,
20263            e2e_defer_reason: None,
20264            progressed: false,
20265            vote_split: false,
20266            reconsideration: Vec::new(),
20267            verdict: None,
20268        }];
20269        // The point: `gate` has not recorded anything yet.
20270        state.gate = Vec::new();
20271        state.gate_ran = false;
20272        state.status = RunStatus::Gating;
20273
20274        let mut runner = Runner {
20275            state,
20276            roles: ResolvedRoles {
20277                implementers: Vec::new(),
20278                judges: Vec::new(),
20279                reviewers: Vec::new(),
20280                fixer: None,
20281                conductor: conductor(),
20282                implementer_roster: Vec::new(),
20283                judge_roster: Vec::new(),
20284                reviewer_roster: Vec::new(),
20285            },
20286            sem: Arc::new(Semaphore::new(1)),
20287            pause: Pause::new(),
20288            interrupt: Pause::new(),
20289        };
20290
20291        runner.merge().await.expect("merge");
20292
20293        assert!(
20294            runner.state.merge.is_none(),
20295            "an empty gate must never be read as a passing one: {:?}",
20296            runner.state.merge
20297        );
20298    }
20299
20300    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
20301    /// commands configured and `merge.mode` is `none` (a review-only run).
20302    /// `gate` must still record a real attempt — zero commands, vacuously
20303    /// passed — rather than leaving `state.gate` empty in a way `merge`
20304    /// cannot tell apart from "never ran"; otherwise the run reaches
20305    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
20306    #[tokio::test]
20307    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20308        let tmp = tempfile::tempdir().expect("tempdir");
20309        let repo = tmp.path().join("repo");
20310        std::fs::create_dir_all(&repo).unwrap();
20311        init_repo(&repo);
20312
20313        // Default config: `verify.gate` empty, `merge.mode` is `none`.
20314        let config = Config::default();
20315
20316        let mut state = RunState::new(
20317            repo.clone(),
20318            "main".to_owned(),
20319            "deadbeef".to_owned(),
20320            "task".to_owned(),
20321            config,
20322        );
20323        state.candidates = vec![Candidate {
20324            index: 0,
20325            label: 'A',
20326            agent: "alpha".to_owned(),
20327            branch: "does-not-exist".to_owned(),
20328            worktree: repo.clone(),
20329            summary: String::new(),
20330            stat: String::new(),
20331            files: 0,
20332            commits: 0,
20333            empty: false,
20334            failed: None,
20335            verified_noop: None,
20336            duration_ms: 0,
20337            folded: false,
20338        }];
20339        state.tally = Some(Tally {
20340            first_choice: BTreeMap::from([('A', 1)]),
20341            borda: BTreeMap::new(),
20342            winner: 'A',
20343            rankings: 1,
20344            unanimous_initial: true,
20345            deliberated: false,
20346            changed_votes: 0,
20347            unanimous_final: true,
20348            tie_break: None,
20349            judges: 0,
20350            present: 0,
20351            quorum: 0,
20352            met_quorum: true,
20353            uncontested: Some("only candidate A produced a change".to_owned()),
20354        });
20355        state.reviews = vec![ReviewRound {
20356            round: 1,
20357            head: "deadbeef".to_owned(),
20358            verified_head: None,
20359            verified_at: None,
20360            reviews: Vec::new(),
20361            e2e: Vec::new(),
20362            fix: None,
20363            blocking: 0,
20364            answered: 0,
20365            expected: 0,
20366            clean: true,
20367            verify_retried: false,
20368            e2e_deferred: false,
20369            e2e_defer_reason: None,
20370            progressed: false,
20371            vote_split: false,
20372            reconsideration: Vec::new(),
20373            verdict: None,
20374        }];
20375
20376        let mut runner = Runner {
20377            state,
20378            roles: ResolvedRoles {
20379                implementers: Vec::new(),
20380                judges: Vec::new(),
20381                reviewers: Vec::new(),
20382                fixer: None,
20383                conductor: conductor(),
20384                implementer_roster: Vec::new(),
20385                judge_roster: Vec::new(),
20386                reviewer_roster: Vec::new(),
20387            },
20388            sem: Arc::new(Semaphore::new(1)),
20389            pause: Pause::new(),
20390            interrupt: Pause::new(),
20391        };
20392
20393        runner.gate().await.expect("gate");
20394        assert!(
20395            runner.state.gate_ran,
20396            "zero configured commands is still a real attempt, not an unrun gate"
20397        );
20398        assert!(runner.state.gate.is_empty());
20399        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20400        assert_ne!(
20401            runner.state.status,
20402            RunStatus::Blocked,
20403            "a gate with nothing to check must not read as failed"
20404        );
20405
20406        runner.merge().await.expect("merge");
20407        assert_eq!(
20408            runner.state.status,
20409            RunStatus::Ready,
20410            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20411        );
20412    }
20413
20414    /// `Config::cache_dir` is derived from `verify.e2e` as well as
20415    /// `verify.gate` (so the e2e leg and the final gate never build against
20416    /// different directories). With zero `verify.gate` commands but a
20417    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
20418    /// that lease before discovering it had nothing to run - so a repo with
20419    /// no gate commands could come back `resource_blocked` (and therefore
20420    /// still `gate_ran == false`) on nothing but an unrelated run holding the
20421    /// cache, exactly the contention this run's own zero commands could
20422    /// never have touched. `gate` must recognise there is nothing to check
20423    /// before it ever asks for the lease.
20424    #[tokio::test]
20425    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20426        crate::run::pin_test_home();
20427        let home = crate::run::home();
20428
20429        let tmp = tempfile::tempdir().expect("tempdir");
20430        let repo = tmp.path().join("repo");
20431        std::fs::create_dir_all(&repo).unwrap();
20432        init_repo(&repo);
20433        // Unique to this test, so holding its lease cannot collide with
20434        // another test sharing the same process-wide `home`.
20435        let cache_dir = tmp.path().join("target");
20436
20437        let mut config = Config::default();
20438        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20439        // `verify.gate` stays empty (the default). Any accidental cache
20440        // acquisition must report contention immediately, regardless of how
20441        // long unrelated Git subprocesses take under parallel test load.
20442        config.graph.timeout_verify = Some(2);
20443        config.graph.timeout_resource_wait = 0;
20444
20445        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20446        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20447            .expect("no io error acquiring directly")
20448        {
20449            crate::cache::AcquireOutcome::Acquired(g) => g,
20450            crate::cache::AcquireOutcome::Busy(b) => {
20451                panic!("expected the direct acquire to win the lease first: {b:?}")
20452            }
20453        };
20454
20455        let mut state = RunState::new(
20456            repo.clone(),
20457            "main".to_owned(),
20458            "deadbeef".to_owned(),
20459            "task".to_owned(),
20460            config,
20461        );
20462        state.candidates = vec![Candidate {
20463            index: 0,
20464            label: 'A',
20465            agent: "alpha".to_owned(),
20466            branch: "does-not-exist".to_owned(),
20467            worktree: repo.clone(),
20468            summary: String::new(),
20469            stat: String::new(),
20470            files: 0,
20471            commits: 0,
20472            empty: false,
20473            failed: None,
20474            verified_noop: None,
20475            duration_ms: 0,
20476            folded: false,
20477        }];
20478        state.tally = Some(Tally {
20479            first_choice: BTreeMap::from([('A', 1)]),
20480            borda: BTreeMap::new(),
20481            winner: 'A',
20482            rankings: 1,
20483            unanimous_initial: true,
20484            deliberated: false,
20485            changed_votes: 0,
20486            unanimous_final: true,
20487            tie_break: None,
20488            judges: 0,
20489            present: 0,
20490            quorum: 0,
20491            met_quorum: true,
20492            uncontested: Some("only candidate A produced a change".to_owned()),
20493        });
20494        state.reviews = vec![ReviewRound {
20495            round: 1,
20496            head: "deadbeef".to_owned(),
20497            verified_head: None,
20498            verified_at: None,
20499            reviews: Vec::new(),
20500            e2e: Vec::new(),
20501            fix: None,
20502            blocking: 0,
20503            answered: 0,
20504            expected: 0,
20505            clean: true,
20506            verify_retried: false,
20507            e2e_deferred: false,
20508            e2e_defer_reason: None,
20509            progressed: false,
20510            vote_split: false,
20511            reconsideration: Vec::new(),
20512            verdict: None,
20513        }];
20514
20515        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20516        state.reviews[0].e2e = vec![CommandOutcome {
20517            command: state.config.verify.e2e[0].clone(),
20518            code: Some(0),
20519            output_tail: String::new(),
20520            duration_ms: 0,
20521            resource_blocked: false,
20522        }];
20523        let mut runner = Runner {
20524            state,
20525            roles: ResolvedRoles {
20526                implementers: Vec::new(),
20527                judges: Vec::new(),
20528                reviewers: Vec::new(),
20529                fixer: None,
20530                conductor: conductor(),
20531                implementer_roster: Vec::new(),
20532                judge_roster: Vec::new(),
20533                reviewer_roster: Vec::new(),
20534            },
20535            sem: Arc::new(Semaphore::new(1)),
20536            pause: Pause::new(),
20537            interrupt: Pause::new(),
20538        };
20539
20540        runner.gate().await.expect("gate");
20541        assert!(runner.state.resource_block.is_none());
20542        assert!(
20543            !runner
20544                .state
20545                .events
20546                .iter()
20547                .any(|event| { event.message.contains("waiting for the shared build cache") }),
20548            "a gate with nothing to run must never request the held cache lease"
20549        );
20550        assert!(
20551            runner.state.gate_ran,
20552            "zero commands is still a real, immediate attempt"
20553        );
20554        assert!(runner.state.gate.is_empty());
20555        assert_ne!(
20556            runner.state.status,
20557            RunStatus::Blocked,
20558            "must not read as resource-blocked on a lease it never asked for"
20559        );
20560    }
20561
20562    /// A separate process carries a live worker's environment throughout all
20563    /// verification entry points; the parallel harness never changes its env.
20564    #[cfg(unix)]
20565    #[tokio::test]
20566    async fn verification_is_standalone_inside_a_live_worker() {
20567        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20568        if std::env::var_os(CHILD).is_none() {
20569            let home = tempfile::tempdir().unwrap();
20570            let generation = crate::worker::pin_current(home.path()).unwrap();
20571            let output = std::process::Command::new(std::env::current_exe().unwrap())
20572                .args([
20573                    "--exact",
20574                    "graph::tests::verification_is_standalone_inside_a_live_worker",
20575                    "--nocapture",
20576                ])
20577                .env(CHILD, "1")
20578                .env("MAGI_HOME", home.path())
20579                .env("MAGI_WORKER_HOME", home.path())
20580                .env("MAGI_WORKER_ID", "abcdef")
20581                .env("MAGI_WORKER_GENERATION", &generation.id)
20582                .env("MAGI_WORKER_EXE", &generation.executable)
20583                .env("MAGI_RUN", "verification-parent")
20584                .env("MAGI_NODE", "gate")
20585                .env("MAGI_SEAT", "worker-fixture")
20586                .env("MAGI_PROCESS_OWNER", "fixture-owner")
20587                .env("MAGI_NO_AUTOUPDATE", "0")
20588                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20589                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20590                .env(
20591                    "VERIFY_EXPECT_TOOLCHAIN",
20592                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20593                )
20594                .env(
20595                    "VERIFY_EXPECT_CACHE",
20596                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20597                )
20598                .output()
20599                .unwrap();
20600            assert!(
20601                output.status.success()
20602                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20603                "{}\n{}",
20604                String::from_utf8_lossy(&output.stdout),
20605                String::from_utf8_lossy(&output.stderr)
20606            );
20607            return;
20608        }
20609        let home = crate::run::home();
20610        let directory = home.join("workers/jobs/abcdef");
20611        std::fs::create_dir_all(&directory).unwrap();
20612        let job = crate::worker::Job {
20613            id: "abcdef".into(),
20614            sequence: 1,
20615            kind: "task".into(),
20616            key: "task:verification".into(),
20617            payload: serde_json::json!({}),
20618            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20619            version: Some(env!("CARGO_PKG_VERSION").into()),
20620            status: crate::worker::Status::Running,
20621            result: None,
20622            pid: Some(std::process::id()),
20623            created_at: 0,
20624            finished_at: None,
20625        };
20626        std::fs::write(
20627            directory.join("job.json"),
20628            serde_json::to_vec(&job).unwrap(),
20629        )
20630        .unwrap();
20631        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20632        let assert_worker = || {
20633            assert_eq!(
20634                crate::worker::current_job(&home).unwrap().unwrap().id,
20635                job.id
20636            );
20637            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20638            assert_eq!(
20639                crate::worker::helper_executable(&home).unwrap(),
20640                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20641            );
20642            assert!(
20643                crate::updater::Checker::new(&crate::config::Update {
20644                    mode: crate::config::UpdateMode::Notify,
20645                    interval: None,
20646                    auto_restart: false,
20647                })
20648                .is_none(),
20649                "the driver remains pinned, even while tests are standalone"
20650            );
20651        };
20652        assert_worker();
20653        let repo = home.join("repo");
20654        std::fs::create_dir_all(&repo).unwrap();
20655        init_repo(&repo);
20656        // Keep storage, seat/control context, containment and tool lookup.
20657        // No inherited values (or credentials) are printed on success.
20658        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20659            test -z "${MAGI_WORKER_GENERATION+x}" &&
20660            test -z "${MAGI_WORKER_HOME+x}" &&
20661            test -z "${MAGI_WORKER_EXE+x}" &&
20662            test "$MAGI_HOME/repo" -ef . &&
20663            test "$MAGI_RUN" = verification-parent &&
20664            test "$MAGI_NODE" = gate &&
20665            test "$MAGI_SEAT" = worker-fixture &&
20666            test -n "$MAGI_PROCESS_OWNER" &&
20667            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20668            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20669            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20670            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20671        // Negative control: an ordinary child inherits the contamination.
20672        assert!(
20673            !std::process::Command::new("sh")
20674                .args(["-c", probe])
20675                .current_dir(&repo)
20676                .status()
20677                .unwrap()
20678                .success()
20679        );
20680        let unrelated_home = tempfile::tempdir().unwrap();
20681        assert!(
20682            crate::worker::current_job(unrelated_home.path()).is_err(),
20683            "an inherited worker id cannot be resolved in a unit fixture's home"
20684        );
20685        let mut runner = runner_at(RunStatus::Gating);
20686        runner.state.repo = repo.clone();
20687        runner.state.config.disk.min_free_bytes = 0;
20688        runner.state.config.coordination = crate::config::test_coordination();
20689        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20690        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20691        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20692        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20693        let winner = Candidate {
20694            index: 0,
20695            label: 'A',
20696            agent: "fixture".into(),
20697            branch: "main".into(),
20698            worktree: repo.clone(),
20699            summary: String::new(),
20700            stat: String::new(),
20701            files: 0,
20702            commits: 0,
20703            empty: false,
20704            failed: None,
20705            verified_noop: None,
20706            duration_ms: 0,
20707            folded: false,
20708        };
20709        let (e2e, retried, pids) = run_e2e_with_retry(
20710            &mut runner.state,
20711            &["sh".into(), "-c".into()],
20712            &[format!("{probe} && printf initial")],
20713            &repo,
20714            Duration::from_secs(10),
20715            "initial",
20716        )
20717        .await;
20718        assert!(!retried && pids.is_empty());
20719        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20720        assert_eq!(e2e[0].output_tail, "initial");
20721        let gate = runner.run_gate(&winner).await.unwrap();
20722        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20723        // An interrupted repair must recheck e2e from saved state. Landing
20724        // reuses that state and must run both pre-gate and gate again.
20725        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20726            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20727            passed: false,
20728            outcomes: Vec::new(),
20729        });
20730        runner.state.save().unwrap();
20731        let mut resumed = RunState::load(&runner.state.id).unwrap();
20732        let wait = crate::repo_lock::WaitPolicy::default();
20733        assert!(matches!(
20734            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20735                .await
20736                .unwrap(),
20737            GateFix::Retry
20738        ));
20739        let saved = RunState::load(&resumed.id).unwrap();
20740        let repair = saved.gate_repair_e2e.unwrap();
20741        assert!(repair.passed);
20742        assert_eq!(repair.outcomes.len(), 1);
20743        assert!(repair.outcomes[0].ok());
20744        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20745            .await
20746            .unwrap();
20747        assert_eq!(landing.len(), 1);
20748        assert!(landing[0].ok(), "{landing:?}");
20749        assert_eq!(
20750            std::fs::read_to_string(repo.join("phases")).unwrap(),
20751            "pre\ngate\ne2e\npre\ngate\n"
20752        );
20753        assert_worker();
20754    }
20755
20756    #[cfg(unix)]
20757    #[tokio::test]
20758    async fn verification_keeps_the_panic_and_the_full_output() {
20759        crate::run::pin_test_home();
20760        let tmp = tempfile::tempdir().unwrap();
20761        let mut config = Config::default();
20762        config.disk.min_free_bytes = 0;
20763        let mut state = RunState::new(
20764            tmp.path().to_owned(),
20765            "main".into(),
20766            "base".into(),
20767            "verification evidence".into(),
20768            config,
20769        );
20770        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20771                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20772                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20773        let mut names = Vec::new();
20774        for _ in 0..2 {
20775            let (outcomes, _) = run_commands(
20776                &mut state,
20777                "gate",
20778                "gate",
20779                0,
20780                &["sh".into(), "-c".into()],
20781                &[script.into()],
20782                tmp.path(),
20783                Duration::from_secs(60),
20784            )
20785            .await;
20786            let tail = &outcomes[0].output_tail;
20787            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20788            assert!(tail.contains("boom assertion"));
20789            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20790            let line = tail.lines().next().unwrap();
20791            names.push(line.to_owned());
20792        }
20793        assert_ne!(names[0], names[1], "the second run must not overwrite");
20794        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20795        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20796        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20797
20798        // Storage failures are evidence too; preserve the command's verdict.
20799        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20800        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20801        let (outcomes, _) = run_commands(
20802            &mut state,
20803            "gate",
20804            "gate",
20805            0,
20806            &["sh".into(), "-c".into()],
20807            &["echo failure-output; exit 7".into()],
20808            tmp.path(),
20809            Duration::from_secs(10),
20810        )
20811        .await;
20812        assert_eq!(outcomes[0].code, Some(7));
20813        assert!(state.events.iter().any(|e| {
20814            e.node == "gate" && e.message.contains("could not save the full gate output:")
20815        }));
20816    }
20817
20818    #[cfg(unix)]
20819    #[tokio::test]
20820    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20821        crate::run::pin_test_home();
20822        let tmp = tempfile::tempdir().unwrap();
20823        let mut config = Config::default();
20824        config.disk.min_free_bytes = 0;
20825        let mut state = RunState::new(
20826            tmp.path().to_owned(),
20827            "main".into(),
20828            "base".into(),
20829            "verification evidence".into(),
20830            config,
20831        );
20832        let (outcomes, pids) = run_commands(
20833            &mut state,
20834            "gate",
20835            "gate",
20836            1,
20837            &["sh".into(), "-c".into()],
20838            &[
20839                "printf before-timeout; exec sleep 30".into(),
20840                "kill -TERM $$".into(),
20841                "exit 7".into(),
20842                "printf success".into(),
20843            ],
20844            tmp.path(),
20845            Duration::from_secs(1),
20846        )
20847        .await;
20848        wait_for_timed_out_children_to_die(&pids).await;
20849        assert_eq!(
20850            pids.len(),
20851            1,
20852            "unleased timeout retains its death-check PID"
20853        );
20854        assert_eq!(outcomes.len(), 4);
20855        assert_eq!(outcomes[0].code, None);
20856        assert!(outcomes[0].output_tail.contains("before-timeout"));
20857        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20858        assert_eq!(outcomes[1].code, None);
20859        assert!(outcomes[1].output_tail.contains("command terminated:"));
20860        assert!(!outcomes[1].output_tail.contains("timed out"));
20861        assert_eq!(outcomes[2].code, Some(7));
20862        assert!(!outcomes[2].output_tail.contains("timed out"));
20863        assert_eq!(outcomes[3].code, Some(0));
20864        assert_eq!(outcomes[3].output_tail, "success");
20865        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20866    }
20867
20868    /// The addendum's second gap: a `verify.gate` command running for real
20869    /// wall-clock time had nothing at all to show for it in `active` before
20870    /// `run_commands` learned to record it — a run could sit in `Gating` for
20871    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20872    /// what was actually happening. Proven with a genuinely still-running
20873    /// command, not just a before/after check on the final state: a poller
20874    /// task reads the same `run.json` `gate()` is writing, the same way the
20875    /// phone or `magi show` would, while the shell command is still blocked
20876    /// on its own release marker.
20877    #[tokio::test]
20878    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20879        crate::run::pin_test_home();
20880
20881        let tmp = tempfile::tempdir().expect("tempdir");
20882        let repo = tmp.path().join("repo");
20883        std::fs::create_dir_all(&repo).unwrap();
20884        init_repo(&repo);
20885
20886        let mut config = Config::default();
20887        config.disk.min_free_bytes = 0;
20888        config.verify.gate = vec![
20889            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20890             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20891                .to_owned(),
20892        ];
20893
20894        let mut state = RunState::new(
20895            repo.clone(),
20896            "main".to_owned(),
20897            "deadbeef".to_owned(),
20898            "task".to_owned(),
20899            config,
20900        );
20901        let run_id = state.id.clone();
20902        state.candidates = vec![Candidate {
20903            index: 0,
20904            label: 'A',
20905            agent: "alpha".to_owned(),
20906            branch: "does-not-exist".to_owned(),
20907            worktree: repo.clone(),
20908            summary: String::new(),
20909            stat: String::new(),
20910            files: 0,
20911            commits: 0,
20912            empty: false,
20913            failed: None,
20914            verified_noop: None,
20915            duration_ms: 0,
20916            folded: false,
20917        }];
20918        state.tally = Some(Tally {
20919            first_choice: BTreeMap::from([('A', 1)]),
20920            borda: BTreeMap::new(),
20921            winner: 'A',
20922            rankings: 1,
20923            unanimous_initial: true,
20924            deliberated: false,
20925            changed_votes: 0,
20926            unanimous_final: true,
20927            tie_break: None,
20928            judges: 0,
20929            present: 0,
20930            quorum: 0,
20931            met_quorum: true,
20932            uncontested: Some("only candidate A produced a change".to_owned()),
20933        });
20934        state.reviews = vec![ReviewRound {
20935            round: 1,
20936            head: "deadbeef".to_owned(),
20937            verified_head: None,
20938            verified_at: None,
20939            reviews: Vec::new(),
20940            e2e: Vec::new(),
20941            fix: None,
20942            blocking: 0,
20943            answered: 0,
20944            expected: 0,
20945            clean: true,
20946            verify_retried: false,
20947            e2e_deferred: false,
20948            e2e_defer_reason: None,
20949            progressed: false,
20950            vote_split: false,
20951            reconsideration: Vec::new(),
20952            verdict: None,
20953        }];
20954
20955        let mut runner = Runner {
20956            state,
20957            roles: ResolvedRoles {
20958                implementers: Vec::new(),
20959                judges: Vec::new(),
20960                reviewers: Vec::new(),
20961                fixer: None,
20962                conductor: conductor(),
20963                implementer_roster: Vec::new(),
20964                judge_roster: Vec::new(),
20965                reviewer_roster: Vec::new(),
20966            },
20967            sem: Arc::new(Semaphore::new(1)),
20968            pause: Pause::new(),
20969            interrupt: Pause::new(),
20970        };
20971
20972        let started_marker = repo.join("started.marker");
20973        let release_marker = repo.join("release.marker");
20974        let poller = tokio::spawn(async move {
20975            // Bounded so a regression that never records the task entry
20976            // fails this test in seconds instead of hanging the suite —
20977            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20978            // does_not_cut_it_short` uses for the same reason.
20979            for _ in 0..100 {
20980                if started_marker.exists()
20981                    && let Ok(s) = crate::run::RunState::load(&run_id)
20982                    && let Some(a) = s.active.get("gate")
20983                {
20984                    std::fs::write(&release_marker, b"go").expect("release marker");
20985                    return Some(a.clone());
20986                }
20987                tokio::time::sleep(Duration::from_millis(50)).await;
20988            }
20989            None
20990        });
20991
20992        runner.gate().await.expect("gate");
20993        let captured = poller.await.expect("poller task");
20994        let captured = captured.expect(
20995            "the poller never saw a `gate` task entry in run.json while the command was \
20996             still blocked on its own release marker",
20997        );
20998
20999        assert_eq!(captured.task.as_deref(), Some("gate"));
21000        assert_eq!(captured.node, "gate");
21001        assert_eq!(captured.index, Some(1));
21002        assert_eq!(captured.total, Some(1));
21003        assert!(
21004            captured
21005                .command
21006                .as_deref()
21007                .is_some_and(|c| c.contains("started.marker")),
21008            "{captured:?}"
21009        );
21010
21011        assert!(
21012            runner.state.active.is_empty(),
21013            "the entry must be cleared once the command actually finished: {:?}",
21014            runner.state.active
21015        );
21016        assert!(runner.state.gate_ran);
21017        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
21018    }
21019
21020    /// The hand-off over a blocking finding a reviewer rejected on leaves a
21021    /// record for `land`; one with only a Minor, or no reject, leaves none.
21022    #[tokio::test]
21023    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
21024        use crate::verdict::{Finding, ReviewVote, Severity};
21025        crate::run::pin_test_home();
21026        let tmp = tempfile::tempdir().expect("tempdir");
21027        let repo = tmp.path().join("repo");
21028        std::fs::create_dir_all(&repo).unwrap();
21029        init_repo(&repo);
21030
21031        for (severity, vote, expect) in [
21032            (Severity::Major, ReviewVote::Reject, true),
21033            (Severity::Minor, ReviewVote::Reject, false),
21034            (Severity::Major, ReviewVote::Approve, false),
21035        ] {
21036            let mut round = review_round(false, 1, 1, 1, false, true);
21037            round.reviews = vec![ReviewRecord {
21038                reviewer: 1,
21039                agent: "alpha".to_owned(),
21040                summary: String::new(),
21041                findings: vec![Finding {
21042                    id: "R1-1-1".to_owned(),
21043                    severity,
21044                    file: None,
21045                    line: None,
21046                    title: "t".to_owned(),
21047                    detail: String::new(),
21048                }],
21049                vote: Some(vote),
21050                failed: None,
21051                duration_ms: 0,
21052                attempts: 0,
21053            }];
21054            let mut state = RunState::new(
21055                repo.clone(),
21056                "main".to_owned(),
21057                "deadbeef".to_owned(),
21058                "task".to_owned(),
21059                Config::default(),
21060            );
21061            state.reviews = vec![round];
21062            let mut runner = Runner {
21063                state,
21064                roles: ResolvedRoles {
21065                    implementers: Vec::new(),
21066                    judges: Vec::new(),
21067                    reviewers: Vec::new(),
21068                    fixer: None,
21069                    conductor: conductor(),
21070                    implementer_roster: Vec::new(),
21071                    judge_roster: Vec::new(),
21072                    reviewer_roster: Vec::new(),
21073                },
21074                sem: Arc::new(Semaphore::new(1)),
21075                pause: Pause::new(),
21076                interrupt: Pause::new(),
21077            };
21078            let shell = runner.state.config.shell();
21079            runner
21080                .stop_reviewing("round budget spent", &shell, &repo)
21081                .await
21082                .expect("stop_reviewing");
21083            assert_eq!(runner.state.status, RunStatus::Gating);
21084            assert_eq!(
21085                runner.state.contested_handoff.is_some(),
21086                expect,
21087                "{severity:?} + {vote:?}"
21088            );
21089        }
21090    }
21091
21092    /// The shape the incident this whole fix responds to actually had: the
21093    /// round budget spent, the last round's own e2e blocked on the shared
21094    /// build cache (held here by a live pid — this test process — exactly
21095    /// `cache`'s own unit tests' pattern for "another owner, still alive"
21096    /// without forking a process). `stop_reviewing` must retry it — not
21097    /// silently leave the round looking untouched (the catch-up-only half of
21098    /// the bug), and not read the contention as a red `e2e` and block the
21099    /// run on it (the other half). Called directly, the same way
21100    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
21101    /// above exercises `gate`, so this never needs a real cargo build to
21102    /// reach: the lease is never released, so `with_cache_lease` never gets
21103    /// past acquiring it into anything that would need a real workspace.
21104    #[tokio::test]
21105    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
21106        crate::run::pin_test_home();
21107        let home = crate::run::home();
21108
21109        let tmp = tempfile::tempdir().expect("tempdir");
21110        let repo = tmp.path().join("repo");
21111        std::fs::create_dir_all(&repo).unwrap();
21112        init_repo(&repo);
21113        let head = crate::git::rev_parse(&repo, "HEAD")
21114            .await
21115            .expect("rev-parse");
21116        // Unique to this test, so holding its lease cannot collide with
21117        // another test sharing the same process-wide `home`.
21118        let cache_dir = tmp.path().join("target");
21119
21120        let mut config = Config::default();
21121        config.verify.e2e = vec![format!(
21122            "CARGO_TARGET_DIR='{}' test -f README.md",
21123            cache_dir.display()
21124        )];
21125        config.graph.review_rounds = 1;
21126        // Bounded so a regression that does start waiting fails the test in
21127        // seconds, not hangs it.
21128        config.graph.timeout_verify = Some(2);
21129        config.graph.timeout_resource_wait = 2;
21130
21131        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21132        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21133            .expect("no io error acquiring directly")
21134        {
21135            crate::cache::AcquireOutcome::Acquired(g) => g,
21136            crate::cache::AcquireOutcome::Busy(b) => {
21137                panic!("expected the direct acquire to win the lease first: {b:?}")
21138            }
21139        };
21140
21141        let mut state = RunState::new(
21142            repo.clone(),
21143            "main".to_owned(),
21144            head.clone(),
21145            "task".to_owned(),
21146            config,
21147        );
21148        state.candidates = vec![Candidate {
21149            index: 0,
21150            label: 'A',
21151            agent: "alpha".to_owned(),
21152            branch: "does-not-exist".to_owned(),
21153            worktree: repo.clone(),
21154            summary: String::new(),
21155            stat: String::new(),
21156            files: 0,
21157            commits: 0,
21158            empty: false,
21159            failed: None,
21160            verified_noop: None,
21161            duration_ms: 0,
21162            folded: false,
21163        }];
21164        state.tally = Some(Tally {
21165            first_choice: BTreeMap::from([('A', 1)]),
21166            borda: BTreeMap::new(),
21167            winner: 'A',
21168            rankings: 1,
21169            unanimous_initial: true,
21170            deliberated: false,
21171            changed_votes: 0,
21172            unanimous_final: true,
21173            tie_break: None,
21174            judges: 0,
21175            present: 0,
21176            quorum: 0,
21177            met_quorum: true,
21178            uncontested: Some("only candidate A produced a change".to_owned()),
21179        });
21180        // The round budget's last round, deferred: `needs_catchup_run`'s
21181        // other trigger. `stop_reviewing`'s retry machinery must treat this
21182        // exactly like a resource-blocked attempt once it actually runs.
21183        state.reviews = vec![ReviewRound {
21184            round: 1,
21185            head: head.clone(),
21186            verified_head: None,
21187            verified_at: None,
21188            reviews: Vec::new(),
21189            e2e: Vec::new(),
21190            fix: None,
21191            blocking: 1,
21192            answered: 1,
21193            expected: 1,
21194            clean: false,
21195            verify_retried: false,
21196            e2e_deferred: true,
21197            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
21198            progressed: false,
21199            vote_split: false,
21200            reconsideration: Vec::new(),
21201            verdict: None,
21202        }];
21203
21204        let mut runner = Runner {
21205            state,
21206            roles: ResolvedRoles {
21207                implementers: Vec::new(),
21208                judges: Vec::new(),
21209                reviewers: Vec::new(),
21210                fixer: None,
21211                conductor: conductor(),
21212                implementer_roster: Vec::new(),
21213                judge_roster: Vec::new(),
21214                reviewer_roster: Vec::new(),
21215            },
21216            sem: Arc::new(Semaphore::new(1)),
21217            pause: Pause::new(),
21218            interrupt: Pause::new(),
21219        };
21220
21221        let shell = runner.state.config.shell();
21222        runner
21223            .stop_reviewing("round budget spent", &shell, &repo)
21224            .await
21225            .expect("stop_reviewing");
21226
21227        let last = runner.state.reviews.last().expect("round record");
21228        assert_eq!(
21229            last.e2e_status(),
21230            E2eStatus::ResourceBlocked,
21231            "the shared cache is still held; the attempt must read as blocked, not deferred or \
21232             failed: {last:?}"
21233        );
21234        assert_eq!(
21235            last.verified_head.as_deref(),
21236            Some(head.as_str()),
21237            "which commit this attempt targeted is known even though nothing finished checking \
21238             it"
21239        );
21240        let first_attempt_at = last
21241            .verified_at
21242            .expect("when this attempt ran is known too");
21243        assert_ne!(
21244            runner.state.status,
21245            RunStatus::Blocked,
21246            "contention is evidence about the machine, not the patch — it must not settle the \
21247             run as blocked: {:?}",
21248            runner.state.status
21249        );
21250        assert!(
21251            !runner
21252                .state
21253                .events
21254                .iter()
21255                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
21256            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
21257            runner.state.events
21258        );
21259
21260        // The cache is still held: a later reentry must retry the same
21261        // round's verification again — not leave it looking exactly as
21262        // untouched as the first blocked attempt, which is indistinguishable
21263        // from never having tried again at all.
21264        runner
21265            .stop_reviewing("round budget spent", &shell, &repo)
21266            .await
21267            .expect("stop_reviewing retry");
21268        assert_eq!(
21269            runner.state.reviews.len(),
21270            1,
21271            "no new round was started: {:?}",
21272            runner.state.reviews
21273        );
21274        let last = runner.state.reviews.last().expect("round record");
21275        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
21276        assert!(
21277            last.verified_at.expect("still known") > first_attempt_at,
21278            "a second reentry must be a fresh attempt, not a stale copy of the first"
21279        );
21280        assert_ne!(runner.state.status, RunStatus::Blocked);
21281
21282        held.release();
21283    }
21284
21285    #[cfg(unix)]
21286    #[tokio::test]
21287    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
21288        ask_test_home();
21289        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21290            let tmp = tempfile::tempdir().unwrap();
21291            let repo = tmp.path().join("repo");
21292            std::fs::create_dir(&repo).unwrap();
21293            init_repo(&repo);
21294            let run_git = |args: &[&str]| {
21295                let out = std::process::Command::new("git")
21296                    .args(args)
21297                    .current_dir(&repo)
21298                    .output()
21299                    .unwrap();
21300                assert!(
21301                    out.status.success(),
21302                    "{args:?}: {}",
21303                    String::from_utf8_lossy(&out.stderr)
21304                );
21305            };
21306            let origin = tmp.path().join("origin.git");
21307            run_git(&["init", "--bare", origin.to_str().unwrap()]);
21308            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21309            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21310            run_git(&["add", "spec.rs"]);
21311            run_git(&["commit", "-m", "initial spec"]);
21312            run_git(&["push", "origin", "main"]);
21313            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21314            run_git(&["checkout", "-b", "fix/review"]);
21315            std::fs::write(
21316                repo.join("check.rs"),
21317                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21318            )
21319            .unwrap();
21320            run_git(&["add", "check.rs"]);
21321            run_git(&["commit", "-m", "use spec"]);
21322            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21323            run_git(&["checkout", "main"]);
21324            std::fs::write(
21325                repo.join("spec.rs"),
21326                "struct AgentSpec { max_turn_secs: u64 }\n",
21327            )
21328            .unwrap();
21329            run_git(&["add", "spec.rs"]);
21330            run_git(&["commit", "-m", "extend spec"]);
21331            run_git(&["push", "origin", "main"]);
21332            run_git(&["checkout", "fix/review"]);
21333
21334            let mut runner = runner_at(RunStatus::Reviewing);
21335            runner.state.repo = repo.clone();
21336            runner.state.base_commit = base;
21337            runner.state.pr_head = Some("fix/review".into());
21338            runner.state.config.graph.review_rounds = max_rounds;
21339            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21340            runner.state.config.graph.sessions = false;
21341            runner.state.config.disk.min_free_bytes = 0;
21342            runner.state.config.verify.e2e =
21343                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21344            runner.state.candidates = vec![Candidate {
21345                index: 0,
21346                label: 'A',
21347                agent: "fixture".into(),
21348                branch: "fix/review".into(),
21349                worktree: repo.clone(),
21350                summary: String::new(),
21351                stat: String::new(),
21352                files: 1,
21353                commits: 1,
21354                empty: false,
21355                failed: None,
21356                verified_noop: None,
21357                duration_ms: 0,
21358                folded: false,
21359            }];
21360            runner.state.tally = Some(Tally {
21361                first_choice: BTreeMap::new(),
21362                borda: BTreeMap::new(),
21363                winner: 'A',
21364                rankings: 0,
21365                unanimous_initial: true,
21366                deliberated: false,
21367                changed_votes: 0,
21368                unanimous_final: true,
21369                tie_break: None,
21370                judges: 0,
21371                present: 0,
21372                quorum: 0,
21373                met_quorum: true,
21374                uncontested: Some("review-only".into()),
21375            });
21376            let mut last = review_round(false, 0, 1, 1, false, false);
21377            last.head = reviewed.clone();
21378            last.e2e_deferred = true;
21379            last.e2e[0].resource_blocked = true;
21380            last.e2e[0].output_tail = "waiting for the shared build cache".into();
21381            if max_rounds == 1 {
21382                last.e2e.clear();
21383            }
21384            runner.state.reviews = vec![last];
21385            let calls = tmp.path().join("fix-calls");
21386            let mut fixer = spec("repair");
21387            fixer
21388                .env
21389                .insert("FIX_CALLS".into(), calls.display().to_string());
21390            fixer.command = vec!["sh".into(), "-c".into(), r#"
21391                echo fix >> "$FIX_CALLS"
21392                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21393                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21394                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21395                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21396            "#.into()];
21397            let mut reviewer = spec("reviewer");
21398            reviewer.command = vec![
21399                "sh".into(),
21400                "-c".into(),
21401                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21402                    .into(),
21403            ];
21404            runner.roles.fixer = Some(vec![fixer]);
21405            runner.roles.reviewers = vec![reviewer];
21406
21407            runner.sync_to_base().await.unwrap();
21408            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21409            assert_ne!(rebased, reviewed);
21410            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21411            if reentry {
21412                // Simulate a crash after catch-up evidence was saved but before
21413                // dispatching the fixer. No dedicated pending-repair field exists.
21414                let shell = runner.state.config.shell();
21415                runner
21416                    .stop_unresolved_review("deferred verification", &shell, &repo)
21417                    .await
21418                    .unwrap();
21419                assert!(runner.catchup_repairable());
21420                assert!(!calls.exists());
21421                runner.state = RunState::load(&runner.state.id).unwrap();
21422                assert!(runner.catchup_repairable());
21423            }
21424            runner.review_loop().await.unwrap();
21425            let first = &runner.state.reviews[0];
21426            assert_eq!(first.head, reviewed);
21427            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21428            assert!(
21429                first.verify_retried,
21430                "a reproduced build failure gets exactly one retry: {:?}",
21431                first.e2e
21432            );
21433            assert!(first.e2e[0].output_tail.contains("E0063"));
21434            assert!(!runner.catchup_repairable());
21435            if max_rounds == 2 {
21436                assert_eq!(
21437                    runner.state.status,
21438                    RunStatus::Gating,
21439                    "{}",
21440                    crate::report::run(&runner.state)
21441                );
21442                assert_eq!(runner.state.reviews.len(), 2);
21443                assert!(first.fix.as_ref().unwrap().committed);
21444                assert!(runner.state.reviews[1].clean);
21445                assert_eq!(
21446                    runner
21447                        .state
21448                        .reviews
21449                        .iter()
21450                        .filter(|r| r.fix.is_some())
21451                        .count(),
21452                    1
21453                );
21454                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21455                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21456            } else {
21457                assert_eq!(runner.state.status, RunStatus::Blocked);
21458                assert!(first.fix.is_none());
21459                assert!(!calls.exists());
21460                let recovery = crate::recovery::assess(
21461                    "prior",
21462                    Some(&runner.state),
21463                    &runner.landing_base(),
21464                    crate::recovery::Relation::Diverged,
21465                );
21466                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21467                let reasons = recovery.reasons.join("\n");
21468                assert!(
21469                    reasons.contains(&reviewed) && reasons.contains(&rebased),
21470                    "{reasons}"
21471                );
21472                assert!(
21473                    reasons.contains("base sync to")
21474                        && reasons.contains("additional review is required"),
21475                    "{reasons}"
21476                );
21477            }
21478        }
21479    }
21480
21481    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
21482    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
21483    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
21484    /// own top-of-function fast path (`review_conclusion`) correctly reads
21485    /// this shape as `None` rather than guessing `Blocked`, and the loop's
21486    /// own `for` range is empty once the round budget is spent, so
21487    /// `review_loop` must retry the check itself rather than silently doing
21488    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
21489    /// above exercises directly, but through `review_loop`'s own entry point
21490    /// this time, proving the wiring between the two rather than just the
21491    /// retry logic in isolation.
21492    #[tokio::test]
21493    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21494        crate::run::pin_test_home();
21495        let home = crate::run::home();
21496
21497        let tmp = tempfile::tempdir().expect("tempdir");
21498        let repo = tmp.path().join("repo");
21499        std::fs::create_dir_all(&repo).unwrap();
21500        init_repo(&repo);
21501        let head = crate::git::rev_parse(&repo, "HEAD")
21502            .await
21503            .expect("rev-parse");
21504        let cache_dir = tmp.path().join("target");
21505
21506        let mut config = Config::default();
21507        config.verify.e2e = vec![format!(
21508            "CARGO_TARGET_DIR='{}' test -f README.md",
21509            cache_dir.display()
21510        )];
21511        config.graph.review_rounds = 1;
21512        config.graph.timeout_verify = Some(2);
21513        config.graph.timeout_resource_wait = 2;
21514
21515        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21516        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21517            .expect("no io error acquiring directly")
21518        {
21519            crate::cache::AcquireOutcome::Acquired(g) => g,
21520            crate::cache::AcquireOutcome::Busy(b) => {
21521                panic!("expected the direct acquire to win the lease first: {b:?}")
21522            }
21523        };
21524
21525        let mut state = RunState::new(
21526            repo.clone(),
21527            "main".to_owned(),
21528            head.clone(),
21529            "task".to_owned(),
21530            config,
21531        );
21532        state.candidates = vec![Candidate {
21533            index: 0,
21534            label: 'A',
21535            agent: "alpha".to_owned(),
21536            branch: "does-not-exist".to_owned(),
21537            worktree: repo.clone(),
21538            summary: String::new(),
21539            stat: String::new(),
21540            files: 0,
21541            commits: 0,
21542            empty: false,
21543            failed: None,
21544            verified_noop: None,
21545            duration_ms: 0,
21546            folded: false,
21547        }];
21548        state.tally = Some(Tally {
21549            first_choice: BTreeMap::from([('A', 1)]),
21550            borda: BTreeMap::new(),
21551            winner: 'A',
21552            rankings: 1,
21553            unanimous_initial: true,
21554            deliberated: false,
21555            changed_votes: 0,
21556            unanimous_final: true,
21557            tie_break: None,
21558            judges: 0,
21559            present: 0,
21560            quorum: 0,
21561            met_quorum: true,
21562            uncontested: Some("only candidate A produced a change".to_owned()),
21563        });
21564        // The exact shape a prior process's `stop_reviewing` would have left
21565        // on disk: the round budget's last round, a real attempt already
21566        // made and already resource-blocked.
21567        state.reviews = vec![ReviewRound {
21568            round: 1,
21569            head: head.clone(),
21570            verified_head: Some(head.clone()),
21571            verified_at: Some(jiff::Timestamp::now()),
21572            reviews: Vec::new(),
21573            e2e: vec![CommandOutcome {
21574                command: format!(
21575                    "CARGO_TARGET_DIR='{}' test -f README.md",
21576                    cache_dir.display()
21577                ),
21578                code: None,
21579                output_tail: "waiting for the shared build cache".to_owned(),
21580                duration_ms: 0,
21581                resource_blocked: true,
21582            }],
21583            fix: None,
21584            blocking: 1,
21585            answered: 1,
21586            expected: 1,
21587            clean: false,
21588            verify_retried: false,
21589            e2e_deferred: false,
21590            e2e_defer_reason: None,
21591            progressed: false,
21592            vote_split: false,
21593            reconsideration: Vec::new(),
21594            verdict: None,
21595        }];
21596
21597        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21598        let mut runner = Runner {
21599            state,
21600            roles: ResolvedRoles {
21601                implementers: Vec::new(),
21602                judges: Vec::new(),
21603                reviewers: Vec::new(),
21604                fixer: None,
21605                conductor: conductor(),
21606                implementer_roster: Vec::new(),
21607                judge_roster: Vec::new(),
21608                reviewer_roster: Vec::new(),
21609            },
21610            sem: Arc::new(Semaphore::new(1)),
21611            pause: Pause::new(),
21612            interrupt: Pause::new(),
21613        };
21614
21615        // The lease is still held throughout, so this reentry's own retry is
21616        // also contended — proving `review_loop` actually tried again (not
21617        // that it happened to succeed) is what the timestamp comparison
21618        // below is for.
21619        runner.review_loop().await.expect("review_loop");
21620
21621        assert_eq!(
21622            runner.state.reviews.len(),
21623            1,
21624            "no new round was started on top of the unresolved one: {:?}",
21625            runner.state.reviews
21626        );
21627        let last = &runner.state.reviews[0];
21628        assert_eq!(
21629            last.e2e_status(),
21630            E2eStatus::ResourceBlocked,
21631            "still contended: {last:?}"
21632        );
21633        assert!(
21634            last.verified_at.expect("still known") > first_attempt_at,
21635            "review_loop must have actually retried the check, not left it exactly as found"
21636        );
21637        assert_ne!(
21638            runner.state.status,
21639            RunStatus::Blocked,
21640            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21641            runner.state.status
21642        );
21643
21644        held.release();
21645    }
21646
21647    #[tokio::test]
21648    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21649        crate::run::pin_test_home();
21650        for status in [RunStatus::Blocked, RunStatus::Gating] {
21651            let mut runner = runner_at(status);
21652            runner.state.config.graph.land = true;
21653            runner.state.gate_ran = true;
21654            runner.state.merge = Some(MergeOutcome {
21655                mode: MergeMode::Pr,
21656                ok: false,
21657                detail: "the fixer produced no commit".into(),
21658                empty: false,
21659            });
21660            let url = "https://example.invalid/o/r/pull/1";
21661            runner.state.pr = Some(crate::run::PrRecord {
21662                url: url.into(),
21663                number: 1,
21664                state: "open".into(),
21665                checks: "red".into(),
21666                round: 1,
21667                rounds: 1,
21668                red_at_merge: Vec::new(),
21669            });
21670            runner.state.save().unwrap();
21671            // The nonexistent repository makes observation fail before a forge
21672            // command can run. We still must reach land, rather than silently
21673            // return from the graph with its old gate and merge records.
21674            runner.execute_graph().await.unwrap();
21675            assert_eq!(runner.state.status, RunStatus::Blocked);
21676            assert!(runner.state.land_sync_pending);
21677            assert!(runner.state.events.iter().any(|event|
21678                event.node == "land" && event.message == format!("watching {url}")));
21679            assert_eq!(
21680                runner.state.merge.as_ref().unwrap().detail,
21681                "the fixer produced no commit"
21682            );
21683        }
21684    }
21685
21686    #[tokio::test]
21687    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21688        crate::run::pin_test_home();
21689        let tmp = tempfile::tempdir().expect("tempdir");
21690        let repo = tmp.path().join("repo");
21691        std::fs::create_dir_all(&repo).unwrap();
21692        init_repo(&repo);
21693
21694        let mut config = Config::default();
21695        config.merge.mode = MergeMode::Pr;
21696        config.graph.land = true;
21697        config.graph.land_approval = false;
21698
21699        let mut state = RunState::new(
21700            repo.clone(),
21701            "main".to_owned(),
21702            "deadbeef".to_owned(),
21703            "task".to_owned(),
21704            config,
21705        );
21706        state.candidates = vec![Candidate {
21707            index: 0,
21708            label: 'A',
21709            agent: "alpha".to_owned(),
21710            branch: "does-not-exist".to_owned(),
21711            worktree: repo.clone(),
21712            summary: String::new(),
21713            stat: String::new(),
21714            files: 0,
21715            commits: 0,
21716            empty: false,
21717            failed: None,
21718            verified_noop: None,
21719            duration_ms: 0,
21720            folded: false,
21721        }];
21722        state.tally = Some(Tally {
21723            first_choice: BTreeMap::from([('A', 1)]),
21724            borda: BTreeMap::new(),
21725            winner: 'A',
21726            rankings: 1,
21727            unanimous_initial: true,
21728            deliberated: false,
21729            changed_votes: 0,
21730            unanimous_final: true,
21731            tie_break: None,
21732            judges: 0,
21733            present: 0,
21734            quorum: 0,
21735            met_quorum: true,
21736            uncontested: Some("only candidate A produced a change".to_owned()),
21737        });
21738        state.reviews = vec![ReviewRound {
21739            round: 1,
21740            head: "deadbeef".to_owned(),
21741            verified_head: None,
21742            verified_at: None,
21743            reviews: Vec::new(),
21744            e2e: Vec::new(),
21745            fix: None,
21746            blocking: 0,
21747            answered: 0,
21748            expected: 0,
21749            clean: true,
21750            verify_retried: false,
21751            e2e_deferred: false,
21752            e2e_defer_reason: None,
21753            progressed: false,
21754            vote_split: false,
21755            reconsideration: Vec::new(),
21756            verdict: None,
21757        }];
21758        state.gate = vec![CommandOutcome {
21759            command: "test".to_owned(),
21760            code: Some(0),
21761            output_tail: String::new(),
21762            duration_ms: 0,
21763            resource_blocked: false,
21764        }];
21765        state.gate_ran = true;
21766        // A first pass through `merge` already pushed and opened this pull
21767        // request; `status` is `Landing` because a previous call into `land`
21768        // parked or was interrupted before it reached a terminal outcome.
21769        state.status = RunStatus::Landing;
21770        state.merge = Some(MergeOutcome {
21771            mode: MergeMode::Pr,
21772            ok: true,
21773            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21774            empty: false,
21775        });
21776
21777        // The Landing-resume shortcut calls `run_land` directly rather than
21778        // through `merge`, which is exactly the call site that used to skip
21779        // `settle_questions` - see the fixture below.
21780        ask_test_home();
21781        // A real resume loads a persisted run; driver ownership is checked
21782        // against that record before the Landing shortcut is entered.
21783        state.save().expect("save landing run");
21784        let store = ask::Questions::open();
21785        let q = ask_open_question(&store, &state.id);
21786
21787        let mut runner = Runner {
21788            state,
21789            roles: ResolvedRoles {
21790                implementers: Vec::new(),
21791                judges: Vec::new(),
21792                reviewers: Vec::new(),
21793                fixer: None,
21794                conductor: conductor(),
21795                implementer_roster: Vec::new(),
21796                judge_roster: Vec::new(),
21797                reviewer_roster: Vec::new(),
21798            },
21799            sem: Arc::new(Semaphore::new(1)),
21800            pause: Pause::new(),
21801            interrupt: Pause::new(),
21802        };
21803
21804        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21805        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21806        // exactly because `review_loop` would otherwise clobber the marker
21807        // first.
21808        runner.execute().await.expect("execute");
21809
21810        assert_eq!(
21811            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21812            Some("https://example.invalid/x/y/pull/1"),
21813            "reentry must not push again or open a second pull request over the \
21814             one `land` is already watching"
21815        );
21816        assert_ne!(
21817            runner.state.status,
21818            RunStatus::Landing,
21819            "land could not actually reach the fake pull request, so it must \
21820             have given up rather than left the run silently parked forever"
21821        );
21822        // `land` could not reach the fake pull request, so it gave up into
21823        // `Blocked` - still resumable, so the question must not have been
21824        // swept just because this branch now also calls `settle_questions`.
21825        assert_eq!(runner.state.status, RunStatus::Blocked);
21826        assert!(
21827            store.get(&q.id).unwrap().status.open(),
21828            "Blocked is still alive; settle_questions must have been a no-op here"
21829        );
21830    }
21831
21832    fn state_with_round(round: ReviewRound) -> RunState {
21833        let mut s = RunState::new(
21834            PathBuf::from("/repo"),
21835            "main".to_owned(),
21836            "abc1234".to_owned(),
21837            "add retries".to_owned(),
21838            Config::default(),
21839        );
21840        s.reviews = vec![round];
21841        s
21842    }
21843
21844    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21845        crate::verdict::Finding {
21846            id: id.to_owned(),
21847            severity,
21848            file: None,
21849            line: None,
21850            title: title.to_owned(),
21851            detail: String::new(),
21852        }
21853    }
21854
21855    #[test]
21856    fn pr_body_names_open_findings_and_declined_ones() {
21857        let round = ReviewRound {
21858            round: 2,
21859            head: "deadbee".to_owned(),
21860            verified_head: None,
21861            verified_at: None,
21862            reviews: vec![ReviewRecord {
21863                attempts: 0,
21864                reviewer: 1,
21865                agent: "alpha".to_owned(),
21866                summary: String::new(),
21867                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21868                vote: None,
21869                failed: None,
21870                duration_ms: 0,
21871            }],
21872            e2e: vec![CommandOutcome {
21873                command: "cargo test".to_owned(),
21874                code: Some(0),
21875                output_tail: String::new(),
21876                duration_ms: 0,
21877                resource_blocked: false,
21878            }],
21879            verify_retried: false,
21880            e2e_deferred: false,
21881            e2e_defer_reason: None,
21882            fix: Some(FixRecord {
21883                agent: "alpha".to_owned(),
21884                addressed: Vec::new(),
21885                rejected: vec![crate::verdict::Rejection {
21886                    id: "R1-1-1".to_owned(),
21887                    why: "not reachable from any caller".to_owned(),
21888                }],
21889                notes: String::new(),
21890                committed: true,
21891                failed: None,
21892                duration_ms: 0,
21893                continuation: None,
21894            }),
21895            blocking: 0,
21896            answered: 1,
21897            expected: 1,
21898            clean: false,
21899            progressed: true,
21900            vote_split: false,
21901            reconsideration: Vec::new(),
21902            verdict: None,
21903        };
21904        let state = state_with_round(round);
21905        let body = pr_message(&state, 'A').body;
21906
21907        assert!(body.contains("add retries"), "the task must still be there");
21908        assert!(body.contains("R2-1-1"), "{body}");
21909        assert!(!body.contains("unused import"), "{body}");
21910        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21911        assert!(
21912            !body.contains("not reachable from any caller"),
21913            "the reason it was declined: {body}"
21914        );
21915    }
21916
21917    #[test]
21918    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21919        let round = ReviewRound {
21920            round: 1,
21921            head: "deadbee".to_owned(),
21922            verified_head: None,
21923            verified_at: None,
21924            reviews: vec![ReviewRecord {
21925                attempts: 0,
21926                reviewer: 1,
21927                agent: "alpha".to_owned(),
21928                summary: String::new(),
21929                findings: Vec::new(),
21930                vote: None,
21931                failed: None,
21932                duration_ms: 0,
21933            }],
21934            e2e: Vec::new(),
21935            verify_retried: false,
21936            e2e_deferred: false,
21937            e2e_defer_reason: None,
21938            fix: None,
21939            blocking: 0,
21940            answered: 1,
21941            expected: 1,
21942            clean: true,
21943            progressed: false,
21944            vote_split: false,
21945            reconsideration: Vec::new(),
21946            verdict: None,
21947        };
21948        let state = state_with_round(round);
21949        let body = pr_message(&state, 'A').body;
21950        assert!(!body.contains("Open review findings"), "{body}");
21951        assert!(!body.contains("Declined"), "{body}");
21952    }
21953
21954    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21955        let mut state = RunState::new(
21956            PathBuf::from("/repo"),
21957            "main".to_owned(),
21958            "abc1234".to_owned(),
21959            instruction.to_owned(),
21960            Config::default(),
21961        );
21962        // These fixtures test execution flow, independent of host disk capacity.
21963        state.config.disk.min_free_bytes = 0;
21964        state.candidates.push(Candidate {
21965            index: 0,
21966            label: 'A',
21967            agent: "alpha".to_owned(),
21968            branch: "magi/x/A".to_owned(),
21969            worktree: PathBuf::from("/wt"),
21970            summary: summary.to_owned(),
21971            stat: String::new(),
21972            files: 1,
21973            commits: 1,
21974            empty: false,
21975            failed: None,
21976            verified_noop: None,
21977            folded: false,
21978            duration_ms: 0,
21979        });
21980        state
21981    }
21982
21983    fn review_state(subjects: &[&str]) -> RunState {
21984        let mut state = state_with_summary(
21985            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21986            "",
21987        );
21988        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21989        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21990        state
21991    }
21992
21993    /// A branch rebased onto a main that moved past the recorded
21994    /// `base_commit` is titled from its own first commit, never main's.
21995    #[tokio::test]
21996    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21997        ask_test_home();
21998        let tmp = tempfile::tempdir().unwrap();
21999        let repo = tmp.path().join("repo");
22000        std::fs::create_dir_all(&repo).unwrap();
22001        init_repo(&repo);
22002        let origin = tmp.path().join("origin.git");
22003        let g = |dir: &Path, args: &[&str]| {
22004            let out = std::process::Command::new("git")
22005                .args(args)
22006                .current_dir(dir)
22007                .quiet()
22008                .output()
22009                .expect("spawn git");
22010            assert!(
22011                out.status.success(),
22012                "git {args:?}: {}",
22013                String::from_utf8_lossy(&out.stderr)
22014            );
22015        };
22016        g(
22017            tmp.path(),
22018            &[
22019                "clone",
22020                "--bare",
22021                "-q",
22022                repo.to_str().unwrap(),
22023                origin.to_str().unwrap(),
22024            ],
22025        );
22026        g(
22027            &repo,
22028            &["remote", "add", "origin", origin.to_str().unwrap()],
22029        );
22030        let c1 = git::rev_parse(&repo, "main").await.unwrap();
22031
22032        // Main moves on; the branch is built on top of the new main.
22033        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
22034        g(&repo, &["add", "-A"]);
22035        g(
22036            &repo,
22037            &["commit", "-q", "-m", "chore(deps): update a crate"],
22038        );
22039        g(&repo, &["push", "-q", "origin", "main"]);
22040        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
22041        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22042        g(&repo, &["add", "-A"]);
22043        g(
22044            &repo,
22045            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
22046        );
22047        g(&repo, &["checkout", "-q", "main"]);
22048
22049        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
22050        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
22051        // Without a readable tracking ref the recorded base's merge base is used.
22052        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
22053        assert_eq!(fallback, c1);
22054
22055        let mut state = review_state(&[
22056            "chore(deps): update a crate",
22057            "fix(daemon): apply a chosen action",
22058        ]);
22059        state.repo = repo.clone();
22060        state.base_branch = "main".to_owned();
22061        state.base_commit = c1;
22062        refresh_reviewed_commits(&mut state, "feat/own").await;
22063        assert_eq!(
22064            state.reviewed_commits,
22065            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
22066        );
22067        assert_eq!(
22068            review_title(&state).as_deref(),
22069            Some("fix(daemon): apply a chosen action")
22070        );
22071        assert_eq!(
22072            leaked_subjects(&state, "feat/own").await,
22073            Some(vec!["chore(deps): update a crate".to_owned()])
22074        );
22075        // A stale tracking ref is still used when the fetch fails, but the
22076        // leak list is withheld.
22077        state.config.merge.remote = "nowhere".to_owned();
22078        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
22079    }
22080
22081    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
22082    /// with `origin/main` one commit behind it, and a summarizing agent that
22083    /// prints whatever `reply` holds.
22084    struct SummaryFixture {
22085        _tmp: tempfile::TempDir,
22086        repo: PathBuf,
22087        reply: PathBuf,
22088        runner: Runner,
22089    }
22090
22091    fn summary_fixture() -> SummaryFixture {
22092        ask_test_home();
22093        let tmp = tempfile::tempdir().unwrap();
22094        let repo = tmp.path().join("repo");
22095        std::fs::create_dir_all(&repo).unwrap();
22096        init_repo(&repo);
22097        let origin = tmp.path().join("origin.git");
22098        let g = |args: &[&str]| {
22099            let out = std::process::Command::new("git")
22100                .args(args)
22101                .current_dir(&repo)
22102                .quiet()
22103                .output()
22104                .expect("spawn git");
22105            assert!(
22106                out.status.success(),
22107                "git {args:?}: {}",
22108                String::from_utf8_lossy(&out.stderr)
22109            );
22110        };
22111        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
22112        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
22113        g(&["push", "-q", "origin", "main"]);
22114        g(&["checkout", "-q", "-b", "feat/own"]);
22115        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22116        g(&["add", "-A"]);
22117        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
22118        let reply = tmp.path().join("reply.txt");
22119        std::fs::write(
22120            &reply,
22121            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
22122        )
22123        .unwrap();
22124        let mut runner = runner_at(RunStatus::Gating);
22125        let mut state = review_state(&["feat(own): add the own file"]);
22126        state.repo = repo.clone();
22127        state.base_branch = "main".to_owned();
22128        state.candidates[0].branch = "feat/own".to_owned();
22129        state.candidates[0].worktree = repo.clone();
22130        let mut author = spec("alpha");
22131        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
22132        author
22133            .env
22134            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
22135        state.config.agents = vec![author];
22136        runner.state = state;
22137        SummaryFixture {
22138            _tmp: tmp,
22139            repo,
22140            reply,
22141            runner,
22142        }
22143    }
22144
22145    async fn posted(f: &mut SummaryFixture) -> PrMessage {
22146        let winner = f.runner.state.candidates[0].clone();
22147        f.runner
22148            .guarded_pr_message(&winner, None, true)
22149            .await
22150            .unwrap()
22151            .expect("an English description never parks")
22152    }
22153
22154    fn clean_round_with_minor_findings() -> ReviewRound {
22155        let mut round = passing_round(2, None);
22156        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
22157        round.reviews = vec![ReviewRecord {
22158            attempts: 0,
22159            reviewer: 1,
22160            agent: "alpha".into(),
22161            summary: "内部レビュー".into(),
22162            findings: vec![
22163                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
22164                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
22165            ],
22166            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
22167            failed: None,
22168            duration_ms: 0,
22169        }];
22170        round
22171    }
22172
22173    #[tokio::test]
22174    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
22175        for review_only in [false, true] {
22176            let mut f = summary_fixture();
22177            if !review_only {
22178                f.runner.state.candidates[0].agent = "alpha".into();
22179            }
22180            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
22181            assert!(f.runner.state.reviews[0].clean);
22182            assert!(f.runner.state.reviews[0].fix.is_none());
22183            assert!(f.runner.state.open_findings().is_empty());
22184            assert_eq!(
22185                crate::pr_outcome::required_finding_ids(&f.runner.state),
22186                ["R2-1-1", "R2-1-2"]
22187            );
22188            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22189            let evidence = crate::pr_outcome::summary_evidence(
22190                &f.runner.state,
22191                crate::pr_outcome::FinalHead::At(&head),
22192            );
22193            assert!(
22194                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
22195            );
22196            // The ordinary reply omits both IDs: no cached description is adopted.
22197            let omitted = posted(&mut f).await;
22198            assert!(omitted.body.contains("No description"));
22199            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
22200            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
22201            assert!(f.runner.state.change_summary.is_none());
22202            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
22203            let described = posted(&mut f).await;
22204            assert!(described.body.contains("improve documentation for callers"));
22205            assert!(described.body.contains("remaining edge case"));
22206            assert!(described.body.contains("## Open review findings"));
22207            assert!(!described.body.contains("未修正"));
22208        }
22209    }
22210
22211    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
22212        let mut f = summary_fixture();
22213        if !review_only {
22214            f.runner.state.candidates[0].agent = "alpha".into();
22215        }
22216        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22217        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
22218        fixed
22219            .fix
22220            .as_mut()
22221            .unwrap()
22222            .rejected
22223            .push(crate::verdict::Rejection {
22224                id: "R1-1-1".into(),
22225                why: "呼び出されないコードのため辞退".into(),
22226            });
22227        let mut clean = clean_round_with_minor_findings();
22228        clean.verified_head = Some(head.clone());
22229        f.runner.state.reviews = vec![fixed, clean];
22230        ok_gate(&mut f.runner.state, Some(&head));
22231        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
22232        f.runner.state.config.agents.push(scripted("beta", reply));
22233        f.runner.state.config.roles.language_judge =
22234            Some(crate::config::AgentChoice::One("beta".into()));
22235        f
22236    }
22237
22238    #[tokio::test]
22239    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
22240        for review_only in [false, true] {
22241            let mut f = rewrite_fixture_with_findings(review_only,
22242                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
22243            let m = posted(&mut f).await;
22244            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22245            let expected =
22246                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
22247            assert!(m.body.contains(&expected));
22248            assert!(m.body.contains("## Review rounds"));
22249            assert!(
22250                m.body
22251                    .contains("Gate (`verify.gate`): passed on the final commit")
22252            );
22253            assert!(
22254                m.body
22255                    .contains("Tests (`verify.e2e`): passed on the final commit")
22256            );
22257            let c = f.runner.state.change_summary.as_ref().unwrap();
22258            assert!(
22259                m.body
22260                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
22261            );
22262            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22263            assert!(crate::pr_outcome::covers_required_findings(
22264                &f.runner.state,
22265                &m.body
22266            ));
22267            assert!(m.body.contains("documentation remains incomplete"));
22268            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
22269            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
22270            assert!(f.runner.state.github_text.is_none());
22271        }
22272    }
22273
22274    #[tokio::test]
22275    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
22276        for review_only in [false, true] {
22277            for reply in [
22278                "TITLE: fix: public description\nA short English description.",
22279                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
22280            ] {
22281                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
22282                let winner = f.runner.state.candidates[0].clone();
22283                let posted = f
22284                    .runner
22285                    .guarded_pr_message(&winner, None, true)
22286                    .await
22287                    .unwrap();
22288                assert!(posted.is_none());
22289                assert!(f.runner.state.parked);
22290                let withheld = f.runner.state.github_text.as_ref().unwrap();
22291                assert!(
22292                    withheld.rewrite.is_none(),
22293                    "an incomplete rewrite must not be selectable"
22294                );
22295                let q = ask::Questions::open()
22296                    .get(withheld.question.as_ref().unwrap())
22297                    .unwrap();
22298                assert!(
22299                    !q.choices
22300                        .contains(&crate::github_text::USE_REWRITE.to_owned())
22301                );
22302                assert!(
22303                    q.detail.contains("R1-1-1")
22304                        && q.detail.contains("R2-1-1")
22305                        && q.detail.contains("R2-1-2")
22306                );
22307            }
22308        }
22309    }
22310
22311    #[tokio::test]
22312    async fn japanese_reports_stay_local_in_both_public_run_modes() {
22313        for review_only in [false, true] {
22314            let mut f = summary_fixture();
22315            if !review_only {
22316                f.runner.state.candidates[0].agent = "alpha".into();
22317            }
22318            f.runner.state.instruction = "内部タスクの説明".into();
22319            f.runner.state.candidates[0].summary = "内部の候補報告".into();
22320            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22321            f.runner.state.reviews = vec![
22322                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22323                round_with_notes(2, Some("二回目の内部修正報告です。")),
22324                passing_round(3, Some(&head)),
22325            ];
22326            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22327            ok_gate(&mut f.runner.state, Some(&head));
22328            let captured = f.repo.join("public-prompt.txt");
22329            let author = &mut f.runner.state.config.agents[0];
22330            author.command = vec![
22331                "sh".into(),
22332                "-c".into(),
22333                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22334            ];
22335            author.env.insert(
22336                "PROMPT_FILE".into(),
22337                captured.to_string_lossy().into_owned(),
22338            );
22339            let m = posted(&mut f).await;
22340            assert!(m.body.contains("Adds `own.txt`"));
22341            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22342            assert!(m.body.contains("R1-1-1"));
22343            assert!(
22344                m.body
22345                    .contains("Gate (`verify.gate`): passed on the final commit")
22346            );
22347            assert!(
22348                m.body
22349                    .contains("Tests (`verify.e2e`): passed on the final commit")
22350            );
22351            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22352            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22353            let prompt = std::fs::read_to_string(captured).unwrap();
22354            assert!(prompt.contains(&format!(
22355                "git diff {} {head}",
22356                f.runner.state.change_summary.as_ref().unwrap().base
22357            )));
22358            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22359            assert_eq!(
22360                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22361                "二回目の内部修正報告です。"
22362            );
22363        }
22364    }
22365
22366    #[tokio::test]
22367    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22368        for review_only in [false, true] {
22369            let mut f = summary_fixture();
22370            if !review_only {
22371                f.runner.state.candidates[0].agent = "alpha".into();
22372            }
22373            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22374            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22375            std::fs::write(&f.reply, "no usable description").unwrap();
22376            let failed = posted(&mut f).await;
22377            assert!(
22378                failed
22379                    .body
22380                    .contains("No description of what this change does could be generated")
22381            );
22382            assert!(failed.body.is_ascii());
22383            assert!(f.runner.state.github_text.is_none());
22384            assert!(f.runner.state.change_summary.is_none());
22385            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22386            let evidence = crate::pr_outcome::summary_evidence(
22387                &f.runner.state,
22388                crate::pr_outcome::FinalHead::At(&head),
22389            );
22390            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22391            assert!(
22392                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22393                    .exists()
22394            );
22395            f.runner.state =
22396                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22397            std::fs::write(
22398                &f.reply,
22399                "TITLE: feat(own): add the own file\nAdds the committed own file.",
22400            )
22401            .unwrap();
22402            let retried = posted(&mut f).await;
22403            assert!(retried.body.contains("Adds the committed own file."));
22404            assert_eq!(
22405                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22406                "日本語の修正報告"
22407            );
22408        }
22409    }
22410
22411    #[tokio::test]
22412    async fn same_head_verification_and_findings_invalidate_public_prose() {
22413        let mut f = summary_fixture();
22414        posted(&mut f).await;
22415        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22416        let original = f
22417            .runner
22418            .state
22419            .change_summary
22420            .as_ref()
22421            .unwrap()
22422            .evidence
22423            .clone();
22424        ok_gate(&mut f.runner.state, Some(&head));
22425        std::fs::write(
22426            &f.reply,
22427            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22428        )
22429        .unwrap();
22430        let verified = posted(&mut f).await;
22431        assert!(verified.body.contains("now verified"));
22432        assert_ne!(
22433            f.runner.state.change_summary.as_ref().unwrap().evidence,
22434            original
22435        );
22436        let mut round = round_with_notes(1, Some("内部報告"));
22437        round.reviews.push(ReviewRecord {
22438            attempts: 0,
22439            reviewer: 1,
22440            agent: "alpha".into(),
22441            summary: "内部レビュー".into(),
22442            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22443            vote: None,
22444            failed: None,
22445            duration_ms: 0,
22446        });
22447        round
22448            .fix
22449            .as_mut()
22450            .unwrap()
22451            .rejected
22452            .push(crate::verdict::Rejection {
22453                id: "R1-1-1".into(),
22454                why: "呼び出し元から到達できない".into(),
22455            });
22456        f.runner.state.reviews = vec![round];
22457        // A description omitting the required ID is unusable, even with a title.
22458        let omitted = posted(&mut f).await;
22459        assert!(omitted.body.contains("No description"));
22460        assert!(omitted.body.contains("R1-1-1"));
22461        assert!(!omitted.body.contains("到達"));
22462        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22463        let described = posted(&mut f).await;
22464        assert!(described.body.contains("unreachable from callers"));
22465        assert!(described.body.contains("R1-1-1"));
22466        assert!(!described.body.contains("到達"));
22467        let evidence = f
22468            .runner
22469            .state
22470            .change_summary
22471            .as_ref()
22472            .unwrap()
22473            .evidence
22474            .clone();
22475        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22476        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22477        assert!(posted(&mut f).await.body.contains("rationale updated"));
22478        assert_ne!(
22479            f.runner.state.change_summary.as_ref().unwrap().evidence,
22480            evidence
22481        );
22482        // Old serialized summaries have no evidence fingerprint and must regenerate.
22483        f.runner
22484            .state
22485            .change_summary
22486            .as_mut()
22487            .unwrap()
22488            .evidence
22489            .clear();
22490        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22491        assert!(posted(&mut f).await.body.contains("current specification"));
22492    }
22493
22494    #[tokio::test]
22495    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22496        let mut f = summary_fixture();
22497        let m = posted(&mut f).await;
22498        assert_eq!(m.title, "feat(own): add the own file");
22499        assert!(
22500            m.body
22501                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22502            "{}",
22503            m.body
22504        );
22505        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22506        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22507        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22508        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22509        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22510        let kept = f.runner.state.change_summary.as_ref().unwrap();
22511        assert_eq!(
22512            (kept.head.as_str(), kept.base.as_str()),
22513            (head.as_str(), base.as_str())
22514        );
22515        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22516    }
22517
22518    #[tokio::test]
22519    async fn review_only_summary_receives_the_configured_github_language() {
22520        let mut f = summary_fixture();
22521        f.runner.state.config.graph.github_language = "ja".into();
22522        let captured = f.repo.join("summary-prompt.txt");
22523        let author = &mut f.runner.state.config.agents[0];
22524        author.command = vec![
22525            "sh".into(),
22526            "-c".into(),
22527            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22528        ];
22529        author.env.insert(
22530            "PROMPT_FILE".into(),
22531            captured.to_string_lossy().into_owned(),
22532        );
22533        std::fs::write(
22534            &f.reply,
22535            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22536        )
22537        .unwrap();
22538        let m = posted(&mut f).await;
22539        let prompt = std::fs::read_to_string(captured).unwrap();
22540        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22541        assert!(!prompt.contains("in English"));
22542        assert_eq!(m.title, "feat(own): ファイルを追加");
22543        assert!(m.body.contains("own.txt を追加します。"));
22544    }
22545
22546    #[tokio::test]
22547    async fn review_fixes_come_after_the_whole_change_summary() {
22548        let mut f = summary_fixture();
22549        f.runner.state.reviews = vec![
22550            round_with_notes(1, Some("handled the PATH case")),
22551            round_with_notes(2, None),
22552        ];
22553        let m = posted(&mut f).await;
22554        let summary = m.body.find("## Summary").expect("summary");
22555        let fixes = m.body.find("## Review rounds").expect("review rounds");
22556        assert!(summary < fixes, "{}", m.body);
22557        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22558        assert!(m.body.contains("## Final verification"), "{}", m.body);
22559    }
22560
22561    #[tokio::test]
22562    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22563        let mut f = summary_fixture();
22564        let first = posted(&mut f).await;
22565        let first_head = f.runner.state.change_summary.clone().unwrap().head;
22566        // The same head reuses the record: a different reply is not read.
22567        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22568        let again = posted(&mut f).await;
22569        assert_eq!((again.title, again.body), (first.title, first.body));
22570
22571        let g = |args: &[&str]| {
22572            let out = std::process::Command::new("git")
22573                .args(args)
22574                .current_dir(&f.repo)
22575                .quiet()
22576                .output()
22577                .expect("spawn git");
22578            assert!(out.status.success(), "git {args:?}");
22579        };
22580        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22581        g(&["add", "-A"]);
22582        g(&["commit", "-q", "-m", "fix(own): handle more"]);
22583        let second = posted(&mut f).await;
22584        let kept = f.runner.state.change_summary.clone().unwrap();
22585        assert_ne!(kept.head, first_head);
22586        assert!(second.body.contains("Not read yet."), "{}", second.body);
22587        assert!(!second.body.contains("so the first summary is read"));
22588        assert_eq!(second.title, "feat(own): other");
22589    }
22590
22591    #[tokio::test]
22592    async fn an_unavailable_summary_is_stated_and_never_invented() {
22593        let mut f = summary_fixture();
22594        std::fs::write(&f.reply, "no title line here").unwrap();
22595        let m = posted(&mut f).await;
22596        assert!(
22597            m.body
22598                .contains("No description of what this change does could be generated"),
22599            "{}",
22600            m.body
22601        );
22602        assert!(m.body.contains("internal reports remain in local records"));
22603        assert!(f.runner.state.change_summary.is_none());
22604        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22605        // A preview asks nobody and claims nothing.
22606        let winner = f.runner.state.candidates[0].clone();
22607        let preview = f
22608            .runner
22609            .guarded_pr_message(&winner, None, false)
22610            .await
22611            .unwrap()
22612            .unwrap();
22613        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22614    }
22615
22616    #[test]
22617    fn pr_message_review_single_commit_uses_its_subject() {
22618        let state = review_state(&["feat(nats): per-role user"]);
22619        let m = pr_message(&state, 'A');
22620        assert_eq!(m.title, "feat(nats): per-role user");
22621        assert!(!m.body.contains("Review the work already"), "{}", m.body);
22622        assert!(m.body.contains("## Commits under review"), "{}", m.body);
22623    }
22624
22625    #[test]
22626    fn pr_message_review_multi_commit_takes_the_oldest() {
22627        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22628        let m = pr_message(&state, 'A');
22629        assert_eq!(m.title, "feat: the change");
22630        for s in ["feat: the change", "fix: typo", "fix: again"] {
22631            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22632        }
22633    }
22634
22635    #[test]
22636    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22637        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22638            let mut state = review_state(&[first, "fix: later fixup"]);
22639            state.candidates[0].branch = "機能/ログイン".to_owned();
22640            let m = pr_message(&state, 'A');
22641            assert!(
22642                m.title.starts_with("chore: land candidate A of run"),
22643                "{}",
22644                m.title
22645            );
22646        }
22647    }
22648
22649    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22650        BranchFacts {
22651            commits: commits
22652                .iter()
22653                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22654                .collect(),
22655            stat: stat.to_owned(),
22656        }
22657    }
22658
22659    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22660        let mut r = review_round(true, 0, 1, 1, true, true);
22661        r.round = round;
22662        r.fix = notes.map(|n| FixRecord {
22663            agent: "fixer".to_owned(),
22664            addressed: Vec::new(),
22665            rejected: Vec::new(),
22666            notes: n.to_owned(),
22667            committed: true,
22668            failed: None,
22669            duration_ms: 0,
22670            continuation: None,
22671        });
22672        r
22673    }
22674
22675    #[test]
22676    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22677        let state = review_state(&["ignored"]);
22678        let f = facts(
22679            &[
22680                (
22681                    "fix(login): resolve PATH on macOS",
22682                    "Login shells skip rc files.",
22683                ),
22684                ("fix: address review", ""),
22685            ],
22686            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22687        );
22688        let m = pr_message_with(&state, 'A', Some(&f));
22689        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22690        assert!(
22691            m.body
22692                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22693            "{}",
22694            m.body
22695        );
22696        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22697        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22698        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22699        for banned in [
22700            "Review the work already",
22701            "no task statement",
22702            "Original task",
22703        ] {
22704            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22705        }
22706        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22707    }
22708
22709    #[test]
22710    fn pr_message_review_truncates_a_huge_first_commit_body() {
22711        let state = review_state(&["ignored"]);
22712        let f = facts(
22713            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22714            "s",
22715        );
22716        let m = pr_message_with(&state, 'A', Some(&f));
22717        assert!(m.body.len() < 30_000, "{}", m.body.len());
22718        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22719        assert!(
22720            m.body.contains("- ... 1 more commit(s)"),
22721            "{}",
22722            m.body.len()
22723        );
22724        assert!(m.body.ends_with("magi:candidate-a\n"));
22725    }
22726
22727    #[test]
22728    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22729        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22730        assert_eq!(m.title, "feat: x");
22731        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22732        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22733        assert!(!m.body.contains("no task statement"), "{}", m.body);
22734    }
22735
22736    #[test]
22737    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22738        let mut state = review_state(&["日本語の件名"]);
22739        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22740        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22741        state.candidates[0].branch = "機能/ログイン".to_owned();
22742        assert!(
22743            pr_message(&state, 'A')
22744                .title
22745                .starts_with("chore: land candidate A")
22746        );
22747    }
22748
22749    #[test]
22750    fn pr_message_review_fixes_survive_a_clean_final_round() {
22751        let mut state = review_state(&["feat: x"]);
22752        state.reviews = vec![
22753            round_with_notes(1, Some("handled the PATH case")),
22754            round_with_notes(2, None),
22755        ];
22756        let body = pr_message(&state, 'A').body;
22757        assert!(
22758            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22759            "{body}"
22760        );
22761        assert!(body.contains("### Round 1"), "{body}");
22762
22763        state.reviews = vec![
22764            round_with_notes(1, Some("first fix")),
22765            round_with_notes(2, Some("")),
22766            round_with_notes(3, Some("second fix")),
22767            round_with_notes(4, None),
22768        ];
22769        let body = pr_message(&state, 'A').body;
22770        assert!(
22771            body.contains("### Round 1") && !body.contains("first fix"),
22772            "{body}"
22773        );
22774        assert!(
22775            body.contains("### Round 3") && !body.contains("second fix"),
22776            "{body}"
22777        );
22778        assert!(body.contains("### Round 2"), "{body}");
22779    }
22780
22781    const FINAL: &str = "abcdef1234567890";
22782
22783    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22784        let mut r = round_with_notes(round, None);
22785        r.verified_head = verified.map(str::to_owned);
22786        r
22787    }
22788
22789    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22790        state.gate = vec![CommandOutcome {
22791            command: "gate".to_owned(),
22792            code: Some(0),
22793            output_tail: String::new(),
22794            duration_ms: 0,
22795            resource_blocked: false,
22796        }];
22797        state.gate_ran = true;
22798        state.gate_head = head.map(str::to_owned);
22799    }
22800
22801    fn body_at(state: &RunState) -> String {
22802        pr_message_raw(
22803            state,
22804            'A',
22805            None,
22806            crate::pr_outcome::FinalHead::At(FINAL),
22807            ChangeNote::NotAsked,
22808        )
22809        .body
22810    }
22811
22812    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22813
22814    #[test]
22815    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22816        let mut state = review_state(&["feat: x"]);
22817        state.reviews = vec![
22818            round_with_notes(1, Some(SANDBOXED)),
22819            passing_round(2, Some(FINAL)),
22820        ];
22821        ok_gate(&mut state, Some(FINAL));
22822        let body = body_at(&state);
22823        assert!(!body.contains("Handled the PATH case."), "{body}");
22824        assert!(!body.contains("could not commit"), "{body}");
22825        assert!(!body.contains("could not run"), "{body}");
22826        assert!(
22827            body.contains("The fix is committed on the branch"),
22828            "{body}"
22829        );
22830        assert!(!body.contains("sandbox"), "{body}");
22831        assert!(
22832            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22833            "{body}"
22834        );
22835        assert!(
22836            body.contains("Gate (`verify.gate`): passed on the final commit"),
22837            "{body}"
22838        );
22839        // The raw report is untouched in the record.
22840        assert!(
22841            state.reviews[0]
22842                .fix
22843                .as_ref()
22844                .unwrap()
22845                .notes
22846                .contains("could not commit")
22847        );
22848    }
22849
22850    #[test]
22851    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22852        let mut state = review_state(&["feat: x"]);
22853        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22854        r1.fix.as_mut().unwrap().committed = false;
22855        let mut r2 = passing_round(2, Some(FINAL));
22856        r2.e2e[0].code = Some(1);
22857        state.reviews = vec![r1, r2];
22858        let body = body_at(&state);
22859        assert!(!body.contains("could not commit"), "{body}");
22860        assert!(!body.contains("could not run"), "{body}");
22861        assert!(body.contains("This round produced no commit"), "{body}");
22862        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22863        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22864        assert!(!body.contains("passed"), "{body}");
22865    }
22866
22867    #[test]
22868    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22869        let mut state = review_state(&["feat: x"]);
22870        state.reviews = vec![
22871            round_with_notes(1, Some(SANDBOXED)),
22872            passing_round(2, Some("0123456789abcdef")),
22873        ];
22874        ok_gate(&mut state, Some("0123456789abcdef"));
22875        let body = body_at(&state);
22876        assert!(!body.contains("HTTP tests could not run"), "{body}");
22877        // The commit claim is still settled by the structured record.
22878        assert!(!body.contains("could not commit"), "{body}");
22879        assert!(
22880            body.contains("passed on an earlier commit (`0123456`)"),
22881            "{body}"
22882        );
22883        assert!(!body.contains("passed on the final commit"), "{body}");
22884
22885        // A gate recorded before `gate_head` existed is "not recorded".
22886        ok_gate(&mut state, None);
22887        let body = body_at(&state);
22888        assert!(
22889            body.contains("passed on a commit that was not recorded"),
22890            "{body}"
22891        );
22892    }
22893
22894    #[test]
22895    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22896        let state = state_with_summary(
22897            "add retries to the client",
22898            "TITLE: feat: retries\n\nDid it.",
22899        );
22900        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22901        assert_eq!(m.title, "feat: retries");
22902        assert!(
22903            m.body.contains("<summary>Original task</summary>"),
22904            "{}",
22905            m.body
22906        );
22907        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22908        assert!(
22909            m.body
22910                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22911            "{}",
22912            m.body
22913        );
22914    }
22915
22916    #[test]
22917    fn pr_message_review_bounds_a_long_english_subject() {
22918        let long = format!("feat: {}", "word ".repeat(100));
22919        let m = pr_message(&review_state(&[&long]), 'A');
22920        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22921        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22922    }
22923
22924    #[test]
22925    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22926        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22927        let m = pr_message(&state, 'A');
22928        assert_eq!(m.title, "add retries");
22929        assert!(m.body.contains("<summary>Original task</summary>"));
22930        assert!(!m.body.contains("Commits under review"));
22931        assert_eq!(landing_subject_source(&state), state.instruction);
22932    }
22933
22934    #[test]
22935    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22936        let state = review_state(&["feat: the change", "fix: typo"]);
22937        let source = landing_subject_source(&state);
22938        assert_eq!(land::merge_subject("", &source), "feat: the change");
22939        assert_eq!(
22940            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22941            "feat: the change"
22942        );
22943        // An operator's rename still wins.
22944        assert_eq!(
22945            land::merge_subject("feat: renamed by hand", &source),
22946            "feat: renamed by hand"
22947        );
22948        let blank = review_state(&["日本語"]);
22949        assert!(
22950            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22951        );
22952    }
22953
22954    #[test]
22955    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22956        let state = review_state(&["feat: the change"]);
22957        let source = landing_subject_source(&state);
22958        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22959        assert_eq!(
22960            land::merge_subject(landing_title(&state, old), &source),
22961            "feat: the change"
22962        );
22963        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22964        let task = state_with_summary("add retries", "");
22965        assert_eq!(landing_title(&task, old), old);
22966    }
22967
22968    #[test]
22969    fn pr_message_describes_the_change_not_the_task() {
22970        let state = state_with_summary(
22971            "今回やってほしいこと: results projector を直す",
22972            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22973        );
22974        let m = pr_message(&state, 'A');
22975        assert_eq!(m.title, "fix(web): batch the runs list reads");
22976        assert!(
22977            m.body.starts_with("## Summary\n\n- reads run.json once"),
22978            "{}",
22979            m.body
22980        );
22981        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22982        let task_at = m.body.find("今回やってほしいこと").unwrap();
22983        let details_at = m.body.find("<details>").unwrap();
22984        assert!(
22985            details_at < task_at,
22986            "the task lives inside <details>: {}",
22987            m.body
22988        );
22989        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22990        assert!(m.body.contains("magi:candidate-a"));
22991    }
22992
22993    #[test]
22994    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22995        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22996        let m = pr_message(&state, 'A');
22997        assert_eq!(m.title, "add retries");
22998        assert!(
22999            m.body.contains("## Summary\n\n- did some things"),
23000            "{}",
23001            m.body
23002        );
23003
23004        let none = RunState::new(
23005            PathBuf::from("/repo"),
23006            "main".to_owned(),
23007            "abc1234".to_owned(),
23008            "add retries".to_owned(),
23009            Config::default(),
23010        );
23011        let m = pr_message(&none, 'A');
23012        assert_eq!(m.title, "add retries");
23013        assert!(!m.body.contains("## Summary"), "{}", m.body);
23014    }
23015
23016    #[test]
23017    fn pr_message_refuses_the_candidate_commit_subject() {
23018        for bad in [
23019            "TITLE: magi: candidate A (uncommitted work)",
23020            "TITLE: chore: stuff (uncommitted work)",
23021            "TITLE:   ",
23022        ] {
23023            let state = state_with_summary("add retries", bad);
23024            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
23025        }
23026    }
23027
23028    #[test]
23029    fn pr_message_bounds_a_very_long_task_and_title() {
23030        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
23031        let state = state_with_summary(&long, "- nothing");
23032        let m = pr_message(&state, 'A');
23033        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23034        assert!(!m.title.contains('\n'));
23035
23036        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
23037        let m = pr_message(&state, 'A');
23038        assert!(m.title.starts_with("feat: "));
23039        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23040        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
23041    }
23042
23043    fn long_title_of(instruction: &str) -> String {
23044        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
23045    }
23046
23047    #[test]
23048    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
23049        let first = "Make the landing path keep a readable title for long tasks";
23050        let line = format!(
23051            "{first}. {}",
23052            "And then keep going with more words ".repeat(20)
23053        );
23054        let t = long_title_of(&line);
23055        assert_eq!(t, first);
23056        assert!(!t.starts_with("chore: land"));
23057    }
23058
23059    #[test]
23060    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
23061        let line = "word ".repeat(200);
23062        let t = long_title_of(&line);
23063        assert!(t.ends_with("word..."), "{t}");
23064        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
23065    }
23066
23067    #[test]
23068    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
23069        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
23070            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
23071        }
23072    }
23073
23074    #[test]
23075    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
23076        let mut state = state_with_summary("再試行を追加する", "- nothing");
23077        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
23078        state.config.graph.github_language = "ja".into();
23079        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
23080    }
23081
23082    #[test]
23083    fn pr_message_title_limit_is_exact() {
23084        let at = "a".repeat(PR_TITLE_MAX);
23085        assert_eq!(long_title_of(&at), at);
23086        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
23087        assert!(over.ends_with("..."), "{over}");
23088        assert_eq!(over.chars().count(), PR_TITLE_MAX);
23089    }
23090
23091    #[test]
23092    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
23093        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
23094        let t = long_title_of(&line);
23095        assert!(t.ends_with("..."), "{t}");
23096        assert!(t.is_ascii(), "{t}");
23097    }
23098
23099    #[test]
23100    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
23101        let line = format!(
23102            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
23103            "plus more filler words ".repeat(20)
23104        );
23105        let t = long_title_of(&line);
23106        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
23107    }
23108
23109    #[test]
23110    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
23111        let line = format!("\n\n# {}", "title words ".repeat(40));
23112        let state = state_with_summary(&line, "- nothing");
23113        let m = pr_message(&state, 'A');
23114        assert!(m.title.starts_with("title words"), "{}", m.title);
23115        assert_eq!(
23116            land::merge_subject(&m.title, &landing_subject_source(&state)),
23117            m.title
23118        );
23119        // An operator's rename wins untouched.
23120        assert_eq!(
23121            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
23122            "feat: renamed by hand"
23123        );
23124    }
23125
23126    #[tokio::test]
23127    async fn github_text_rewrite_is_bounded_and_falls_back() {
23128        crate::run::pin_test_home();
23129        for (reply, accepted) in [
23130            (
23131                "TITLE: fix: retries\nAdd retries for failed requests.",
23132                true,
23133            ),
23134            (
23135                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
23136                false,
23137            ),
23138            (
23139                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
23140                true,
23141            ),
23142            (
23143                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
23144                true,
23145            ),
23146            (
23147                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
23148                true,
23149            ),
23150            (
23151                "TITLE: fix: retries\n日本語の説明です。 token=secret",
23152                false,
23153            ),
23154            ("TITLE: fix: retries\n", false),
23155        ] {
23156            let dir = tempfile::tempdir().unwrap();
23157            let mut runner = runner_at(RunStatus::Gating);
23158            runner.state = state_with_summary(
23159                "add retries",
23160                "TITLE: fix: retries\n日本語の説明を書きます。",
23161            );
23162            runner.state.repo = dir.path().to_owned();
23163            runner.state.candidates[0].worktree = dir.path().to_owned();
23164            let mut author = spec("alpha");
23165            author.command = vec![
23166                "sh".into(),
23167                "-c".into(),
23168                "printf '%s' \"$REWRITE_REPLY\"".into(),
23169            ];
23170            author.env.insert("REWRITE_REPLY".into(), reply.into());
23171            runner.state.config.agents = vec![author];
23172            seed_public_summary(&mut runner);
23173            let winner = runner.state.candidates[0].clone();
23174            let first = runner
23175                .guarded_pr_message(&winner, None, true)
23176                .await
23177                .unwrap();
23178            let message = if accepted {
23179                first.expect("a passing rewrite never asks")
23180            } else {
23181                // Withheld: the run parks on a question and posts nothing yet.
23182                assert!(first.is_none());
23183                assert!(runner.state.parked);
23184                let store = ask::Questions::open();
23185                let asked: Vec<_> = store
23186                    .list()
23187                    .into_iter()
23188                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23189                    .collect();
23190                assert_eq!(asked.len(), 1);
23191                assert!(!asked[0].detail.contains("secret"));
23192                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
23193                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
23194                assert_eq!(
23195                    asked[0]
23196                        .choices
23197                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
23198                    rewrite.is_some()
23199                );
23200                // Silence: the question lapses and the neutral text is used,
23201                // without a second rewrite call or a second question.
23202                store
23203                    .update(&asked[0].id, |q| {
23204                        q.abandon("test");
23205                        Ok(())
23206                    })
23207                    .unwrap();
23208                let again = runner
23209                    .guarded_pr_message(&winner, None, true)
23210                    .await
23211                    .unwrap()
23212                    .expect("an abandoned question falls back");
23213                let n = store
23214                    .list()
23215                    .into_iter()
23216                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23217                    .count();
23218                assert_eq!(n, 1, "asked once per rejected text");
23219                again
23220            };
23221            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
23222            assert!(!message.body.contains("secret"));
23223            assert!(!message.body.contains("prefix"));
23224            assert!(!message.title.contains("secret"));
23225            assert_eq!(
23226                message.body.contains("Add retries for failed requests."),
23227                accepted
23228            );
23229            assert_eq!(
23230                runner.state.seats["github-text-rewrite"].turns, 1,
23231                "only one rewrite invocation"
23232            );
23233            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
23234            if accepted {
23235                assert!(runner.state.github_text.is_none());
23236                assert!(!runner.state.parked);
23237                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
23238                assert!(message.body.contains("## Final verification"));
23239                assert!(message.body.contains("Evidence: final diff"));
23240                assert!(
23241                    message
23242                        .body
23243                        .contains(&format!("magi:run/{}", runner.state.id))
23244                );
23245            }
23246            if !accepted {
23247                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23248                assert!(
23249                    message
23250                        .body
23251                        .contains(&format!("magi:run/{}", runner.state.id))
23252                );
23253            }
23254        }
23255    }
23256
23257    fn scripted(id: &str, reply: &str) -> AgentSpec {
23258        let mut a = spec(id);
23259        a.command = vec![
23260            "sh".into(),
23261            "-c".into(),
23262            "printf '%s' \"$REWRITE_REPLY\"".into(),
23263        ];
23264        a.env.insert("REWRITE_REPLY".into(), reply.into());
23265        a
23266    }
23267
23268    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
23269    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
23270
23271    /// Gate tests inject generated public output, not internal reports.
23272    fn seed_public_summary(runner: &mut Runner) {
23273        let repo = &runner.state.repo;
23274        let g = |args: &[&str]| {
23275            let out = std::process::Command::new("git")
23276                .args(args)
23277                .current_dir(repo)
23278                .quiet()
23279                .output()
23280                .unwrap();
23281            assert!(out.status.success(), "git {args:?}");
23282            String::from_utf8_lossy(&out.stdout).trim().to_owned()
23283        };
23284        g(&["init", "-q", "-b", "main"]);
23285        g(&[
23286            "-c",
23287            "user.name=Test",
23288            "-c",
23289            "user.email=test@example.invalid",
23290            "commit",
23291            "-q",
23292            "--allow-empty",
23293            "-m",
23294            "feat: fixture",
23295        ]);
23296        let head = g(&["rev-parse", "HEAD"]);
23297        g(&["update-ref", "refs/remotes/origin/main", &head]);
23298        let evidence = crate::pr_outcome::summary_evidence(
23299            &runner.state,
23300            crate::pr_outcome::FinalHead::At(&head),
23301        );
23302        let generated = &runner.state.candidates[0].summary;
23303        runner.state.change_summary = Some(crate::run::ChangeSummary {
23304            base: head.clone(),
23305            head,
23306            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23307            title: summary_title(generated).unwrap(),
23308            body: summary_without_title(generated),
23309        });
23310    }
23311
23312    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23313        let mut runner = runner_at(RunStatus::Gating);
23314        runner.state = state_with_summary("add retries", JA_SUMMARY);
23315        if review_only {
23316            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23317        }
23318        runner.state.repo = dir.to_owned();
23319        runner.state.candidates[0].worktree = dir.to_owned();
23320        runner.state.config.agents = agents;
23321        seed_public_summary(&mut runner);
23322        runner
23323    }
23324
23325    fn gate_questions(runner: &Runner) -> usize {
23326        ask::Questions::open()
23327            .list()
23328            .into_iter()
23329            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23330            .count()
23331    }
23332
23333    #[tokio::test]
23334    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23335        crate::run::pin_test_home();
23336        let dir = tempfile::tempdir().unwrap();
23337        let mut runner = rewrite_runner(
23338            dir.path(),
23339            true,
23340            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23341        );
23342        runner.state.config.roles.language_judge =
23343            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23344        let winner = runner.state.candidates[0].clone();
23345        let msg = runner
23346            .guarded_pr_message(&winner, None, true)
23347            .await
23348            .unwrap()
23349            .expect("an English rewrite posts without asking");
23350        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23351        assert!(msg.body.contains("<details>kept</details>"));
23352        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23353        assert_eq!(gate_questions(&runner), 0);
23354        assert!(runner.state.github_text.is_none());
23355        assert!(!runner.state.seats.contains_key("impl-A"));
23356        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23357    }
23358
23359    #[tokio::test]
23360    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23361        crate::run::pin_test_home();
23362        let dir = tempfile::tempdir().unwrap();
23363        let mut runner = rewrite_runner(
23364            dir.path(),
23365            false,
23366            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23367        );
23368        runner.state.config.roles.language_judge =
23369            Some(crate::config::AgentChoice::One("beta".into()));
23370        runner.state.seats.insert(
23371            "impl-A".into(),
23372            SeatState::new("impl-A", "alpha", runner.state.seed),
23373        );
23374        let winner = runner.state.candidates[0].clone();
23375        let msg = runner
23376            .guarded_pr_message(&winner, None, true)
23377            .await
23378            .unwrap()
23379            .expect("rewritten");
23380        assert!(msg.body.contains("<details>kept</details>"));
23381        assert_eq!(runner.state.seats["impl-A"].turns, 0);
23382        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23383        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23384    }
23385
23386    #[tokio::test]
23387    async fn rewrite_reuses_a_resumable_implementer_session() {
23388        crate::run::pin_test_home();
23389        let dir = tempfile::tempdir().unwrap();
23390        let mut runner = rewrite_runner(
23391            dir.path(),
23392            false,
23393            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23394        );
23395        runner.state.config.roles.language_judge =
23396            Some(crate::config::AgentChoice::One("beta".into()));
23397        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23398        seat.turns = 1;
23399        runner.state.seats.insert("impl-A".into(), seat);
23400        let winner = runner.state.candidates[0].clone();
23401        runner
23402            .guarded_pr_message(&winner, None, true)
23403            .await
23404            .unwrap()
23405            .expect("rewritten");
23406        assert_eq!(runner.state.seats["impl-A"].turns, 2);
23407        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23408    }
23409
23410    #[tokio::test]
23411    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23412        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
23413        // session, was out of quota, and nothing else was asked.
23414        crate::run::pin_test_home();
23415        let dir = tempfile::tempdir().unwrap();
23416        let mut exhausted = spec("alpha");
23417        exhausted.command = vec!["false".into()];
23418        let mut runner = rewrite_runner(
23419            dir.path(),
23420            false,
23421            vec![exhausted, scripted("beta", EN_REWRITE)],
23422        );
23423        // The implementer's own agent is in the role too: it is not asked twice.
23424        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23425            "alpha".into(),
23426            "beta".into(),
23427        ]));
23428        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23429        seat.turns = 1;
23430        runner.state.seats.insert("impl-A".into(), seat);
23431        let winner = runner.state.candidates[0].clone();
23432        let msg = runner
23433            .guarded_pr_message(&winner, None, true)
23434            .await
23435            .unwrap()
23436            .expect("the role rewrites after the implementer fails");
23437        assert!(msg.body.contains("`src/net.rs`"));
23438        assert_eq!(
23439            runner.state.seats["impl-A"].turns, 2,
23440            "the implementer was tried once"
23441        );
23442        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23443        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23444    }
23445
23446    #[tokio::test]
23447    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23448        crate::run::pin_test_home();
23449        let dir = tempfile::tempdir().unwrap();
23450        let mut broken = spec("bad");
23451        broken.command = vec!["false".into()];
23452        let mut runner = rewrite_runner(
23453            dir.path(),
23454            true,
23455            vec![
23456                broken,
23457                scripted("noformat", "Add retries."),
23458                scripted("good", EN_REWRITE),
23459            ],
23460        );
23461        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23462            "bad".into(),
23463            "noformat".into(),
23464            "good".into(),
23465        ]));
23466        let winner = runner.state.candidates[0].clone();
23467        let msg = runner
23468            .guarded_pr_message(&winner, None, true)
23469            .await
23470            .unwrap()
23471            .expect("the third agent rewrites");
23472        assert!(msg.body.contains("`src/net.rs`"));
23473        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23474        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23475    }
23476
23477    #[tokio::test]
23478    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23479        crate::run::pin_test_home();
23480        // (agents, role): still Japanese; no TITLE; nothing resolvable.
23481        for (agents, role, saved) in [
23482            (
23483                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23484                Some("beta"),
23485                true,
23486            ),
23487            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23488            (Vec::new(), None, false),
23489        ] {
23490            let dir = tempfile::tempdir().unwrap();
23491            let mut runner = rewrite_runner(dir.path(), true, agents);
23492            runner.state.config.roles.language_judge =
23493                role.map(|r| crate::config::AgentChoice::One(r.into()));
23494            let winner = runner.state.candidates[0].clone();
23495            let first = runner
23496                .guarded_pr_message(&winner, None, true)
23497                .await
23498                .unwrap();
23499            assert!(first.is_none(), "withheld text parks the run");
23500            assert!(runner.state.parked);
23501            assert_eq!(gate_questions(&runner), 1);
23502            let g = runner.state.github_text.as_ref().unwrap();
23503            assert_eq!(g.rewrite.is_some(), saved);
23504            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23505        }
23506    }
23507
23508    #[tokio::test]
23509    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23510        crate::run::pin_test_home();
23511        let dir = tempfile::tempdir().unwrap();
23512        let mut runner = runner_at(RunStatus::Gating);
23513        runner.state =
23514            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23515        runner.state.repo = dir.path().to_owned();
23516        runner.state.candidates[0].worktree = dir.path().to_owned();
23517        let mut author = spec("alpha");
23518        author.command = vec![
23519            "sh".into(),
23520            "-c".into(),
23521            "printf '%s' \"$REWRITE_REPLY\"".into(),
23522        ];
23523        author.env.insert(
23524            "REWRITE_REPLY".into(),
23525            format!(
23526                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23527                "English sentence. ".repeat(600)
23528            ),
23529        );
23530        runner.state.config.agents = vec![author];
23531        seed_public_summary(&mut runner);
23532        let winner = runner.state.candidates[0].clone();
23533        assert!(
23534            runner
23535                .guarded_pr_message(&winner, None, true)
23536                .await
23537                .unwrap()
23538                .is_none()
23539        );
23540        let record = runner.state.github_text.clone().unwrap();
23541        let rewrite = record.rewrite.clone().unwrap();
23542        assert_eq!(rewrite.failed, ["body-language"]);
23543        assert!(
23544            rewrite
23545                .body
23546                .ends_with(&format!("magi:run/{}", runner.state.id))
23547        );
23548        let store = ask::Questions::open();
23549        let q = store.get(record.question.as_ref().unwrap()).unwrap();
23550        assert!(
23551            q.choices
23552                .contains(&crate::github_text::USE_REWRITE.to_owned())
23553        );
23554        for label in [
23555            "ORIGINAL",
23556            "FALLBACK",
23557            "REWRITE",
23558            "Rules still failed: body-language",
23559            "truncated",
23560        ] {
23561            assert!(q.detail.contains(label), "{label}");
23562        }
23563        let artifact = std::fs::read_to_string(crate::run::artifact_path(
23564            &runner.state,
23565            &crate::github_text::artifact_name(&record.fingerprint),
23566        ))
23567        .unwrap();
23568        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23569        // Restart from serialized state, then retry after a bad owner title.
23570        runner.state =
23571            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23572        store
23573            .update(&q.id, |q| {
23574                q.say("まだ日本語")?;
23575                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23576            })
23577            .unwrap();
23578        assert!(
23579            runner
23580                .guarded_pr_message(&winner, None, true)
23581                .await
23582                .unwrap()
23583                .is_none()
23584        );
23585        let retry = runner.state.github_text.as_ref().unwrap();
23586        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23587        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23588        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23589        assert!(
23590            q.posting_comparison
23591                .as_ref()
23592                .unwrap()
23593                .matches_detail(&q.detail)
23594        );
23595        store
23596            .update(&q.id, |q| {
23597                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23598            })
23599            .unwrap();
23600        let message = runner
23601            .guarded_pr_message(&winner, None, true)
23602            .await
23603            .unwrap()
23604            .unwrap();
23605        assert_eq!(message.title, rewrite.title);
23606        assert_eq!(message.body, rewrite.body);
23607        assert_eq!(
23608            runner.state.github_text.as_ref().unwrap().choice,
23609            Some(GithubTextChoice::Rewrite)
23610        );
23611        let again = runner
23612            .guarded_pr_message(&winner, None, true)
23613            .await
23614            .unwrap()
23615            .unwrap();
23616        assert_eq!(again.title, message.title);
23617        assert_eq!(again.body, message.body);
23618        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23619        assert!(
23620            !runner
23621                .state
23622                .events
23623                .iter()
23624                .any(|e| e.message.contains(&rewrite.title)
23625                    || e.message.contains("English sentence."))
23626        );
23627    }
23628
23629    #[tokio::test]
23630    async fn historical_posting_reply_never_expires_into_fallback() {
23631        crate::run::pin_test_home();
23632        let dir = tempfile::tempdir().unwrap();
23633        let mut runner = runner_at(RunStatus::Gating);
23634        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23635        runner.state.repo = dir.path().to_owned();
23636        runner.state.candidates[0].worktree = dir.path().to_owned();
23637        seed_public_summary(&mut runner);
23638        let winner = runner.state.candidates[0].clone();
23639        assert!(
23640            runner
23641                .guarded_pr_message(&winner, None, true)
23642                .await
23643                .unwrap()
23644                .is_none()
23645        );
23646        let id = runner
23647            .state
23648            .github_text
23649            .as_ref()
23650            .unwrap()
23651            .question
23652            .clone()
23653            .unwrap();
23654        let store = ask::Questions::open();
23655        let mut q = store.get(&id).unwrap();
23656        q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23657        q.answer(ask::Answer::Choice(
23658            crate::github_text::POST_ORIGINAL.into(),
23659        ))
23660        .unwrap();
23661        q.answer_delivered = true;
23662        store.put(&mut q).unwrap();
23663        let source = std::fs::read(store.path_of(&id)).unwrap();
23664        let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23665        std::fs::create_dir_all(&migration).unwrap();
23666        std::fs::write(
23667            migration.join("state.json"),
23668            serde_json::to_vec(&serde_json::json!({
23669                "schema": 1, "sources": [id], "pending": [id]
23670            }))
23671            .unwrap(),
23672        )
23673        .unwrap();
23674        for _ in 0..2 {
23675            assert!(
23676                runner
23677                    .guarded_pr_message(&winner, None, true)
23678                    .await
23679                    .unwrap()
23680                    .is_none()
23681            );
23682            assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23683            assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23684        }
23685        let home = crate::run::home();
23686        crate::task_chat::sweep(
23687            &crate::queue::Queue::at(home.join("queue")),
23688            &crate::talk::Talks::at(home.join("talks")),
23689            &store,
23690            &home,
23691        );
23692        let outbox = crate::task_inquiry::Outbox::at(&home);
23693        outbox
23694            .update_decision(&id, |d| {
23695                d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23696                d.source_answer_at = q.answered_at;
23697                Ok(true)
23698            })
23699            .unwrap();
23700        outbox
23701            .settle(
23702                &id,
23703                "owner-chat",
23704                "Owner confirms no publication remains necessary",
23705                None,
23706            )
23707            .unwrap();
23708        assert!(
23709            runner
23710                .guarded_pr_message(&winner, None, true)
23711                .await
23712                .unwrap()
23713                .is_none()
23714        );
23715        assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23716        assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23717    }
23718
23719    #[tokio::test]
23720    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23721        crate::run::pin_test_home();
23722        let dir = tempfile::tempdir().unwrap();
23723        let mut runner = runner_at(RunStatus::Gating);
23724        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23725        runner.state.repo = dir.path().to_owned();
23726        runner.state.candidates[0].worktree = dir.path().to_owned();
23727        seed_public_summary(&mut runner);
23728        let winner = runner.state.candidates[0].clone();
23729        assert!(
23730            runner
23731                .guarded_pr_message(&winner, None, true)
23732                .await
23733                .unwrap()
23734                .is_none()
23735        );
23736        let record = runner.state.github_text.as_ref().unwrap();
23737        assert!(record.rewrite.is_none());
23738        // Records written before the additive field deserialize with no rewrite.
23739        let mut old = serde_json::to_value(record).unwrap();
23740        old.as_object_mut().unwrap().remove("rewrite");
23741        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23742        assert!(old.rewrite.is_none());
23743        let id = record.question.clone().unwrap();
23744        ask::Questions::open()
23745            .update(&id, |q| {
23746                q.choices.push(crate::github_text::USE_REWRITE.into());
23747                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23748            })
23749            .unwrap();
23750        let message = runner
23751            .guarded_pr_message(&winner, None, true)
23752            .await
23753            .unwrap()
23754            .unwrap();
23755        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23756        assert_eq!(
23757            runner.state.github_text.as_ref().unwrap().choice,
23758            Some(GithubTextChoice::Fallback)
23759        );
23760    }
23761
23762    #[tokio::test]
23763    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23764        crate::run::pin_test_home();
23765        let dir = tempfile::tempdir().unwrap();
23766        let mut runner = runner_at(RunStatus::Gating);
23767        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23768        runner.state.repo = dir.path().to_owned();
23769        runner.state.candidates[0].worktree = dir.path().to_owned();
23770        seed_public_summary(&mut runner);
23771        let winner = runner.state.candidates[0].clone();
23772        assert!(
23773            runner
23774                .guarded_pr_message(&winner, None, true)
23775                .await
23776                .unwrap()
23777                .is_none()
23778        );
23779        let store = ask::Questions::open();
23780        let find = |run: &str| {
23781            store
23782                .list()
23783                .into_iter()
23784                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23785                .unwrap()
23786        };
23787        let run = runner.state.id.clone();
23788        let q = find(&run);
23789        let fp = runner
23790            .state
23791            .github_text
23792            .as_ref()
23793            .unwrap()
23794            .fingerprint
23795            .clone();
23796        let name = crate::github_text::artifact_name(&fp);
23797        let path = crate::run::artifact_path(&runner.state, &name);
23798        assert!(q.detail.contains(&path.display().to_string()));
23799        assert!(q.detail.contains("title-language: title"));
23800        let saved = std::fs::read_to_string(&path).expect("artifact written");
23801        assert!(saved.contains("日本語のタイトル"));
23802        assert!(q.cwd.is_none());
23803        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23804        // A bad replacement is refused and asked about once more.
23805        let say = |id: &str, text: &str, choice: &str| {
23806            store
23807                .update(id, |q| {
23808                    q.thread.push(ask::Turn {
23809                        who: ask::Who::Operator,
23810                        body: text.to_owned(),
23811                        at: jiff::Timestamp::now(),
23812                        note: None,
23813                    });
23814                    q.answer(ask::Answer::Choice(choice.to_owned()))
23815                })
23816                .unwrap();
23817        };
23818        say(&q.id, "まだ日本語", "use my text");
23819        assert!(
23820            runner
23821                .guarded_pr_message(&winner, None, true)
23822                .await
23823                .unwrap()
23824                .is_none()
23825        );
23826        let second = find(&run);
23827        assert_ne!(second.id, q.id);
23828        assert!(second.detail.contains("did not pass"));
23829        // The retry still compares the original text; the rejected
23830        // replacement is named by category only.
23831        assert!(second.detail.contains("日本語のタイトル"));
23832        assert!(!second.detail.contains("まだ日本語"));
23833        assert!(second.detail.contains(&path.display().to_string()));
23834        say(&second.id, "fix: retry failed requests", "use my text");
23835        let message = runner
23836            .guarded_pr_message(&winner, None, true)
23837            .await
23838            .unwrap()
23839            .expect("a vetted replacement is posted");
23840        assert_eq!(message.title, "fix: retry failed requests");
23841        assert_eq!(
23842            runner
23843                .state
23844                .github_text
23845                .as_ref()
23846                .unwrap()
23847                .chosen_title
23848                .as_deref(),
23849            Some("fix: retry failed requests")
23850        );
23851        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23852        // Resumed again: the saved decision is applied, nothing is re-asked.
23853        let again = runner
23854            .guarded_pr_message(&winner, None, true)
23855            .await
23856            .unwrap()
23857            .unwrap();
23858        assert_eq!(again.title, message.title);
23859        assert_eq!(
23860            store
23861                .list()
23862                .into_iter()
23863                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23864                .count(),
23865            2
23866        );
23867    }
23868
23869    #[test]
23870    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23871        // What magi itself writes stays English under any configured language,
23872        // so a future localisation of these headings fails here. (The agents'
23873        // own text is also checked by the posting gate.)
23874        let mut state = state_with_summary(
23875            "add retries",
23876            "TITLE: fix(web): batch reads\n- reads run.json once",
23877        );
23878        state.config.graph.language = "ja".to_owned();
23879        let m = pr_message(&state, 'A');
23880        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23881        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23882
23883        // The task is the operator's own text: it goes in untouched, and the
23884        // fallback title (no summary) may be in its language too.
23885        let task = "今回やってほしいこと: results projector を直す";
23886        let mut state = state_with_summary(task, "- no title line");
23887        state.config.graph.language = "ja".to_owned();
23888        let m = pr_message(&state, 'A');
23889        assert_eq!(
23890            m.title,
23891            format!("chore: land candidate A of run {}", state.id)
23892        );
23893        assert!(
23894            m.body.contains(&format!(
23895                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23896            )),
23897            "{}",
23898            m.body
23899        );
23900    }
23901
23902    #[test]
23903    fn pr_message_scrubs_home_paths_and_addresses() {
23904        let state = state_with_summary(
23905            "fix it in /Users/someone/src/x",
23906            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23907        );
23908        let m = pr_message(&state, 'A');
23909        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23910            assert!(!m.body.contains(leak), "{}", m.body);
23911        }
23912        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23913    }
23914
23915    #[test]
23916    fn pr_message_survives_a_task_that_closes_details() {
23917        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23918        let m = pr_message(&state, 'A');
23919        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23920    }
23921
23922    #[test]
23923    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23924        let cmd = manual_merge_command(
23925            MergeStyle::Squash,
23926            Path::new("/repo"),
23927            "b",
23928            "fix: \"quoted\" $(x) `y`\n\nbody",
23929        );
23930        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23931    }
23932
23933    #[test]
23934    fn manual_merge_command_matches_the_configured_style() {
23935        let repo = Path::new("/repo");
23936        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23937
23938        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23939        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23940
23941        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23942        assert_eq!(
23943            squash,
23944            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23945             \"Merge magi run 0832 (candidate A)\""
23946        );
23947
23948        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23949        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23950    }
23951
23952    #[test]
23953    fn a_nudge_gets_a_quarter_of_the_budget() {
23954        // The judge and implement budgets magi ships with.
23955        assert_eq!(retry_budget(secs(1200), true), secs(300));
23956        assert_eq!(retry_budget(secs(3600), true), secs(900));
23957    }
23958
23959    #[test]
23960    fn a_resent_prompt_keeps_the_whole_budget() {
23961        // The seat kept no context, so the retry is the original job again and
23962        // shortening it would only guarantee a second failure.
23963        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23964        assert_eq!(retry_budget(secs(60), false), secs(60));
23965    }
23966
23967    #[test]
23968    fn the_floor_never_exceeds_the_original_budget() {
23969        // A short configured timeout must not be *raised* by the floor: the
23970        // operator asked for a bound, and a retry may not outlast the attempt
23971        // it is retrying.
23972        assert_eq!(retry_budget(secs(60), true), secs(60));
23973        assert_eq!(retry_budget(secs(480), true), secs(120));
23974        assert_eq!(retry_budget(secs(0), true), secs(0));
23975    }
23976
23977    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23978        agent::CommandEvidence {
23979            id: "item1".to_owned(),
23980            description: "cargo test".to_owned(),
23981            exit_code,
23982            result_summary: String::new(),
23983            source: "codex".to_owned(),
23984        }
23985    }
23986
23987    #[test]
23988    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23989        // No evidence is not the same fact as unconfirmed evidence: a
23990        // backend with no adapter, or a reply that ran no commands at all,
23991        // must not be misread as carrying a dangling job.
23992        assert!(!has_unconfirmed_command(&[]));
23993    }
23994
23995    #[test]
23996    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23997        // Deliberately not a check on the exit code's *value*: a fixer
23998        // legitimately runs something that fails mid-iteration before it
23999        // succeeds, and that must never by itself reopen a valid report.
24000        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
24001        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
24002        assert!(!has_unconfirmed_command(&[
24003            evidence(Some(0)),
24004            evidence(Some(101))
24005        ]));
24006    }
24007
24008    #[test]
24009    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
24010        assert!(has_unconfirmed_command(&[
24011            evidence(Some(0)),
24012            evidence(None)
24013        ]));
24014    }
24015
24016    #[test]
24017    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
24018        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24019        assert_eq!(
24020            verified_noop_claim(true, &[], text).as_deref(),
24021            Some("already fixed by b32cfc4, on main.")
24022        );
24023    }
24024
24025    #[test]
24026    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
24027        // A timeout or a bad exit code reads as the ordinary loss it is,
24028        // whatever the reply's own prose claims.
24029        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24030        assert!(verified_noop_claim(false, &[], text).is_none());
24031    }
24032
24033    #[test]
24034    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
24035        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24036        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
24037        // A confirmed command alongside the marker is fine.
24038        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
24039    }
24040
24041    #[test]
24042    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
24043        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
24044    }
24045
24046    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
24047    /// pair, in order, labelled A, B, C, ...
24048    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
24049        runner.state.candidates = shape
24050            .iter()
24051            .enumerate()
24052            .map(|(i, &(empty, verified))| Candidate {
24053                index: i,
24054                label: (b'A' + i as u8) as char,
24055                agent: "sonnet".to_owned(),
24056                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
24057                worktree: PathBuf::from(format!("/wt/{i}")),
24058                summary: String::new(),
24059                stat: String::new(),
24060                files: 0,
24061                commits: 0,
24062                empty,
24063                failed: None,
24064                verified_noop: verified.map(str::to_owned),
24065                duration_ms: 0,
24066                folded: false,
24067            })
24068            .collect();
24069    }
24070
24071    #[test]
24072    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
24073        ask_test_home();
24074        let mut runner = runner_at(RunStatus::Implementing);
24075        set_candidates(
24076            &mut runner,
24077            &[
24078                (true, Some("already on main at b32cfc4")),
24079                (true, Some("same fix, see the existing test")),
24080            ],
24081        );
24082
24083        runner
24084            .after_implement()
24085            .expect("a verified no-op is not an error");
24086
24087        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
24088    }
24089
24090    #[test]
24091    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
24092        ask_test_home();
24093        let mut runner = runner_at(RunStatus::Implementing);
24094        // Candidate A declares a verified no-op; candidate B simply wrote
24095        // nothing and said nothing about why. One candidate's claim is not
24096        // the whole run's agreement.
24097        set_candidates(
24098            &mut runner,
24099            &[(true, Some("already on main at b32cfc4")), (true, None)],
24100        );
24101
24102        let err = runner
24103            .after_implement()
24104            .expect_err("an unverified empty candidate must still fail the run");
24105
24106        assert!(
24107            err.to_string().contains("no candidate produced a change"),
24108            "{err}"
24109        );
24110        assert_eq!(runner.state.status, RunStatus::Failed);
24111    }
24112
24113    #[test]
24114    fn after_implement_still_fails_an_ordinary_all_empty_run() {
24115        ask_test_home();
24116        let mut runner = runner_at(RunStatus::Implementing);
24117        set_candidates(&mut runner, &[(true, None), (true, None)]);
24118
24119        let err = runner
24120            .after_implement()
24121            .expect_err("no candidate declared anything; this is an ordinary failure");
24122
24123        assert!(
24124            err.to_string().contains("no candidate produced a change"),
24125            "{err}"
24126        );
24127        assert_eq!(runner.state.status, RunStatus::Failed);
24128    }
24129}
24130
24131/// A graft rollback whose restore could not be confirmed (see
24132/// [`Runner::graft`]).
24133#[derive(Debug)]
24134struct GraftRestoreFailed(String);
24135
24136impl std::fmt::Display for GraftRestoreFailed {
24137    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24138        write!(
24139            f,
24140            "could not restore the winner to its pre-graft commit: {}",
24141            self.0
24142        )
24143    }
24144}
24145
24146impl std::error::Error for GraftRestoreFailed {}
24147
24148/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
24149/// not the agent: they never draw on an agent's own timeout, and running out of
24150/// them never reads as an agent failure.
24151#[cfg(test)]
24152mod resource_wait_tests {
24153    use super::*;
24154    use std::collections::BTreeMap;
24155
24156    fn sh(id: &str, script: &str) -> AgentSpec {
24157        AgentSpec {
24158            id: id.to_owned(),
24159            kind: crate::config::AgentKind::Command,
24160            model: None,
24161            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
24162            extra_args: Vec::new(),
24163            env: BTreeMap::new(),
24164            prompt_delivery: None,
24165            max_turn_secs: None,
24166        }
24167    }
24168
24169    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
24170        SeatJob {
24171            seat: SeatState::new("impl-A", &spec.id, 1),
24172            spec,
24173            cwd: dir.to_path_buf(),
24174            prompt: "do it".to_owned(),
24175            timeout,
24176            allow_write: true,
24177            sessions: false,
24178            artifacts: dir.join("art"),
24179            stem: "impl-A".to_owned(),
24180            handover: None,
24181        }
24182    }
24183
24184    fn state(resource_wait_secs: u64) -> RunState {
24185        let mut config = Config::default();
24186        config.graph.timeout_resource_wait = resource_wait_secs;
24187        RunState::new(
24188            PathBuf::from("/repo"),
24189            "main".to_owned(),
24190            "abc1234".to_owned(),
24191            "task".to_owned(),
24192            config,
24193        )
24194    }
24195
24196    fn hold(cache: &Path) -> crate::cache::Guard {
24197        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
24198        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
24199            crate::cache::AcquireOutcome::Acquired(g) => g,
24200            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
24201        }
24202    }
24203
24204    async fn run(
24205        jobs: Vec<SeatJob>,
24206        sem: Arc<Semaphore>,
24207        cache: Option<&Path>,
24208        st: &mut RunState,
24209    ) -> Vec<(usize, SeatState, AgentOutcome)> {
24210        let prompts = Prompts::default();
24211        let ctx = WaveCtx {
24212            run: "r",
24213            node: "implement",
24214            prompts: &prompts,
24215            cache,
24216            round: None,
24217            carry_seats: false,
24218            pauses: None,
24219        };
24220        wave(jobs, sem, &ctx, st, 0).await
24221    }
24222
24223    #[tokio::test]
24224    async fn verification_cache_wait_preserves_the_command_budget() {
24225        crate::run::pin_test_home();
24226        let tmp = tempfile::tempdir().unwrap();
24227        let cache = tmp.path().join("target");
24228        let guard = hold(&cache);
24229        let release = tokio::spawn(async move {
24230            tokio::time::sleep(Duration::from_millis(100)).await;
24231            guard.release();
24232        });
24233        let mut st = state(10);
24234        let budget = Duration::from_millis(50);
24235        let wait = repository_wait(&st, None);
24236        let (outcomes, _) = with_cache_lease(
24237            &mut st,
24238            Some(&cache),
24239            "gate",
24240            "gate",
24241            tmp.path(),
24242            "abc1234",
24243            budget,
24244            "test gate",
24245            &wait,
24246            |_, available| async move {
24247                assert_eq!(
24248                    available, budget,
24249                    "contention must not shorten verification"
24250                );
24251                (Vec::new(), false, Vec::new())
24252            },
24253        )
24254        .await
24255        .unwrap();
24256        release.await.unwrap();
24257        assert!(outcomes.is_empty(), "{outcomes:?}");
24258        assert!(
24259            st.events
24260                .iter()
24261                .any(|event| event.message.contains("waiting for"))
24262        );
24263        assert!(st.resource_wait.is_none());
24264        // The verification body no longer owns the lease after returning.
24265        hold(&cache).release();
24266    }
24267
24268    #[tokio::test]
24269    async fn verification_cache_wait_expires_without_starting_commands() {
24270        crate::run::pin_test_home();
24271        let tmp = tempfile::tempdir().unwrap();
24272        let cache = tmp.path().join("target");
24273        let _held = hold(&cache);
24274        let mut st = state(0);
24275        let wait = repository_wait(&st, None);
24276        let (outcomes, _) = with_cache_lease(
24277            &mut st,
24278            Some(&cache),
24279            "gate",
24280            "gate",
24281            tmp.path(),
24282            "abc1234",
24283            Duration::from_secs(60),
24284            "test gate",
24285            &wait,
24286            |_, _| async { panic!("commands must not start without the cache lease") },
24287        )
24288        .await
24289        .unwrap();
24290        assert_eq!(outcomes.len(), 1);
24291        assert!(outcomes[0].resource_blocked);
24292        assert!(st.resource_wait.is_none());
24293        assert!(matches!(
24294            st.resource_block.unwrap().cause,
24295            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24296        ));
24297    }
24298
24299    #[tokio::test]
24300    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24301        crate::run::pin_test_home();
24302        let tmp = tempfile::tempdir().unwrap();
24303        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24304        let cache = tmp.path().join("cache");
24305        let mut state = state(21600);
24306        let wait = repository_wait(&state, None);
24307        let (outcomes, _) = with_cache_lease(
24308            &mut state,
24309            Some(&cache),
24310            "gate",
24311            "gate",
24312            tmp.path(),
24313            "head",
24314            Duration::from_secs(60),
24315            "gate",
24316            &wait,
24317            |_, _| async { panic!("freshness failure must not start verification") },
24318        )
24319        .await
24320        .unwrap();
24321        assert_eq!(outcomes.len(), 1);
24322        assert!(outcomes[0].resource_blocked);
24323        let block = state.resource_block.unwrap();
24324        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24325        assert!(!block.description().contains("timed out"));
24326    }
24327
24328    #[tokio::test]
24329    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24330        crate::run::pin_test_home();
24331        let tmp = tempfile::tempdir().unwrap();
24332        let cache = tmp.path().join("cache");
24333        let _held = hold(&cache);
24334        let mut state = state(21600);
24335        let pause = Pause::new();
24336        let interrupt = Pause::new();
24337        let cause = ParkCause::Operator;
24338        pause.park_for(cause.clone());
24339        let prompts = Prompts::default();
24340        let ctx = WaveCtx {
24341            run: "test",
24342            node: "fix",
24343            prompts: &prompts,
24344            cache: Some(&cache),
24345            round: None,
24346            carry_seats: false,
24347            pauses: Some((&pause, &interrupt)),
24348        };
24349        let result = wave(
24350            vec![job(
24351                sh("writer", "exit 99"),
24352                tmp.path(),
24353                Duration::from_secs(1),
24354            )],
24355            Arc::new(Semaphore::new(1)),
24356            &ctx,
24357            &mut state,
24358            0,
24359        )
24360        .await;
24361        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24362        assert_eq!(result[0].1.turns, 0);
24363        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24364        assert_eq!(
24365            state.resource_block.unwrap().cause,
24366            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24367        );
24368        assert!(state.active.is_empty());
24369    }
24370
24371    #[tokio::test]
24372    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24373        crate::run::pin_test_home();
24374        let tmp = tempfile::tempdir().unwrap();
24375        let cache = tmp.path().join("target-a");
24376        let guard = hold(&cache);
24377        // Held for longer than the agent's whole 1s budget.
24378        tokio::spawn(async move {
24379            tokio::time::sleep(Duration::from_millis(1500)).await;
24380            guard.release();
24381        });
24382        let mut st = state(60);
24383        let results = run(
24384            vec![job(
24385                sh("alpha", "echo ok"),
24386                tmp.path(),
24387                Duration::from_secs(1),
24388            )],
24389            Arc::new(Semaphore::new(1)),
24390            Some(&cache),
24391            &mut st,
24392        )
24393        .await;
24394        assert!(
24395            matches!(results[0].2, AgentOutcome::Ok(_)),
24396            "the wait must not be charged to the agent"
24397        );
24398        assert!(st.resource_block.is_none());
24399        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24400        assert!(st.resource_wait.is_none() && st.active.is_empty());
24401    }
24402
24403    #[tokio::test]
24404    #[cfg(unix)]
24405    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24406        use std::os::fd::AsRawFd as _;
24407        crate::run::pin_test_home();
24408        let tmp = tempfile::tempdir().unwrap();
24409        crate::git::git(tmp.path(), &["init"]).await.unwrap();
24410        let common = tmp.path().join(".git");
24411        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24412        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24413        assert_eq!(
24414            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24415            0
24416        );
24417        let mut st = state(1);
24418        st.config.coordination = crate::config::test_coordination();
24419        let mut reader = job(
24420            sh("reader", "echo must-not-run"),
24421            tmp.path(),
24422            Duration::from_secs(30),
24423        );
24424        reader.allow_write = false;
24425        let results = run(
24426            vec![reader.clone()],
24427            Arc::new(Semaphore::new(1)),
24428            None,
24429            &mut st,
24430        )
24431        .await;
24432        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24433        assert_eq!(results[0].1.turns, 0);
24434        assert!(FailClass::of(&results[0].2).is_none());
24435        assert_eq!(
24436            st.resource_block.as_ref().unwrap().kind,
24437            "repository coordination"
24438        );
24439        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24440        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24441        drop(external);
24442        let editor = crate::repo_lock::SeatLease::for_checkout(
24443            tmp.path(),
24444            true,
24445            crate::config::test_coordination().as_ref(),
24446        )
24447        .await
24448        .unwrap();
24449        let pause = Pause::new();
24450        let interrupt = Pause::new();
24451        let prompts = Prompts::default();
24452        let ctx = WaveCtx {
24453            run: "r",
24454            node: "review",
24455            prompts: &prompts,
24456            cache: None,
24457            round: None,
24458            carry_seats: false,
24459            pauses: Some((&pause, &interrupt)),
24460        };
24461        let (results, ()) = tokio::join!(
24462            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24463            async {
24464                tokio::time::sleep(Duration::from_millis(50)).await;
24465                pause.park();
24466            }
24467        );
24468        assert!(
24469            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24470        );
24471        assert_eq!(results[0].1.turns, 0);
24472        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24473        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24474        drop(editor);
24475    }
24476
24477    #[tokio::test]
24478    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24479        crate::run::pin_test_home();
24480        let tmp = tempfile::tempdir().unwrap();
24481        let cache = tmp.path().join("target-b");
24482        let _held = hold(&cache);
24483        let mut st = state(1);
24484        let results = run(
24485            vec![job(
24486                sh("alpha", "echo ok"),
24487                tmp.path(),
24488                Duration::from_secs(1),
24489            )],
24490            Arc::new(Semaphore::new(1)),
24491            Some(&cache),
24492            &mut st,
24493        )
24494        .await;
24495        let (_, seat, out) = &results[0];
24496        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24497        assert!(wave_blocked(&results));
24498        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24499        assert_eq!(seat.turns, 0, "nothing was asked");
24500        assert!(
24501            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24502            "no agent was spawned"
24503        );
24504        assert!(FailClass::of(out).is_none(), "not a failure class");
24505        let block = st.resource_block.as_ref().expect("recorded for the resume");
24506        assert_eq!(block.kind, "shared build cache");
24507        assert!(matches!(
24508            block.cause,
24509            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24510        ));
24511        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24512        assert!(st.active.is_empty());
24513    }
24514
24515    #[tokio::test]
24516    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24517        crate::run::pin_test_home();
24518        let tmp = tempfile::tempdir().unwrap();
24519        let mut st = state(1);
24520        let results = run(
24521            vec![job(
24522                sh("alpha", "echo ok"),
24523                tmp.path(),
24524                Duration::from_secs(1),
24525            )],
24526            Arc::new(Semaphore::new(0)),
24527            None,
24528            &mut st,
24529        )
24530        .await;
24531        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24532        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24533        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24534    }
24535
24536    #[tokio::test]
24537    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24538        crate::run::pin_test_home();
24539        let tmp = tempfile::tempdir().unwrap();
24540        let mut st = state(60);
24541        let results = run(
24542            vec![job(
24543                sh("alpha", "sleep 30"),
24544                tmp.path(),
24545                Duration::from_secs(1),
24546            )],
24547            Arc::new(Semaphore::new(1)),
24548            None,
24549            &mut st,
24550        )
24551        .await;
24552        let out = &results[0].2;
24553        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24554        let class = FailClass::of(out).expect("a failure");
24555        assert_eq!(class, FailClass::Timeout);
24556        assert!(should_hand_over(None, &class), "existing handover policy");
24557        assert!(st.resource_block.is_none());
24558    }
24559}