Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize one bounded synchronization interval, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false, None).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true, None).await
1357    }
1358
1359    pub(crate) async fn authorize_base_recovery_control(
1360        home: &Path,
1361        task: &str,
1362        control: &mut crate::base_recovery_control::Transaction,
1363    ) -> Result<()> {
1364        let id = control.request.run.clone();
1365        Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366    }
1367
1368    async fn authorize_base_recovery_request(
1369        home: &Path,
1370        id: &str,
1371        task: &str,
1372        resumption: bool,
1373        mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374    ) -> Result<()> {
1375        let _owner = crate::pr_head::owner_lock(home, id)?;
1376        let mut state = RunState::load_under(id, home)?;
1377        crate::pr_head::ensure_owner(&state)?;
1378        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379        anyhow::ensure!(
1380            queued.execution_runs().last().is_some_and(|run| run == id)
1381                && queued.status != crate::queue::TaskStatus::Done
1382                && !queued.fresh_start
1383                && queued.review_branch.is_none(),
1384            "run task ownership changed"
1385        );
1386        anyhow::ensure!(
1387            state
1388                .origin
1389                .as_ref()
1390                .and_then(|o| o.task.as_deref())
1391                .is_none_or(|owner| owner == task),
1392            "run belongs to another task"
1393        );
1394        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395        anyhow::ensure!(
1396            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397            "run is live or its ownership is unknown"
1398        );
1399        // Judge the previous driver's lease before acquiring our maintenance
1400        // lease. The owner lock excludes new driver claims across both steps.
1401        let _execution = crate::driver::maintenance(home, id)?;
1402        anyhow::ensure!(
1403            state.status.resumable()
1404                && state.released_to.is_none()
1405                && state.retired_to.is_none()
1406                && state.pr_taken_over_by.is_none(),
1407            "run ownership is final"
1408        );
1409        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410        anyhow::ensure!(
1411            !state.pr_publication_pending
1412                && !crate::handover::landing_checkpoint(&state)
1413                && state.merge_decision.is_none(),
1414            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415        );
1416        let winner = state.winner().context("run has no retained winner")?;
1417        anyhow::ensure!(
1418            git::is_clean(&winner.worktree).await?,
1419            "retained worktree has uncommitted changes"
1420        );
1421        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422        anyhow::ensure!(
1423            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424            "retained worktree does not match the winner branch"
1425        );
1426        if let Some(control) = control.as_ref() {
1427            anyhow::ensure!(
1428                head.starts_with(&control.request.head),
1429                "retained HEAD changed since the owner request"
1430            );
1431            anyhow::ensure!(
1432                state
1433                    .base_sync_recovery
1434                    .stop
1435                    .as_ref()
1436                    .is_none_or(|stop| stop.head == head),
1437                "retained HEAD differs from the recorded stop head; inspect it first"
1438            );
1439        }
1440        state.ensure_base_episode();
1441        state.record_base_sync_stop(&head);
1442        anyhow::ensure!(
1443            state
1444                .base_sync_recovery
1445                .stop
1446                .as_ref()
1447                .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448            "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449        );
1450        let fetched =
1451            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452        anyhow::ensure!(
1453            fetched.ok(),
1454            "cannot authorize base recovery without a fetched base"
1455        );
1456        let approved_tip = git::rev_parse(
1457            &state.repo,
1458            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459        )
1460        .await?;
1461        if let Some(control) = control.as_deref_mut() {
1462            let queue = crate::queue::Queue::at(home.join("queue"));
1463            control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464            if let Some(prior) = state
1465                .base_sync_recovery
1466                .authorizations
1467                .iter()
1468                .find(|a| a.request.as_deref() == Some(&control.key()))
1469            {
1470                anyhow::ensure!(
1471                    !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472                    "recovery request was consumed or invalidated; it cannot queue again"
1473                );
1474                return control.enqueue(&queue);
1475            }
1476        }
1477        if resumption {
1478            anyhow::ensure!(
1479                crate::daemon::compatible_run_instruction(&state, &queued),
1480                "resumption instruction does not match the retained run"
1481            );
1482            let approval = queued
1483                .resumption
1484                .as_ref()
1485                .context("no operator resumption")?;
1486            anyhow::ensure!(
1487                approval.instruction == queued.instruction,
1488                "resumption approval does not match the current task instruction"
1489            );
1490            state.authorize_base_sync_resumption(task, approval)?;
1491        } else {
1492            state.authorize_base_sync(task)?;
1493        }
1494        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495        state
1496            .base_sync_recovery
1497            .authorizations
1498            .last_mut()
1499            .unwrap()
1500            .scope = Some(crate::base_sync_recovery::Scope {
1501            tip: approved_tip,
1502            instruction: queued.instruction.clone(),
1503            episode,
1504        });
1505        if let Some(control) = control.as_deref_mut() {
1506            let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507            authorization.request = Some(control.key());
1508            authorization.generation = control.generation();
1509            authorization.decision = control.decision();
1510            authorization.question = control
1511                .request
1512                .decision
1513                .as_ref()
1514                .map(|_| control.request.request.clone());
1515        }
1516        state.event(
1517            "base-recovery",
1518            "owner authorized a bounded base synchronization interval; fresh verification required",
1519        );
1520        state.save_under(home)?;
1521        if let Some(control) = control {
1522            control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523        }
1524        Ok(())
1525    }
1526
1527    /// Reopen an existing run.
1528    pub fn resume(id: &str) -> Result<Self> {
1529        let state = RunState::load(id)?;
1530        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531        if state.startup.as_ref().is_some_and(|s| !s.complete)
1532            || (state.status == RunStatus::Prep
1533                && state.instruction.trim().is_empty()
1534                && state.candidates.is_empty()
1535                && state.prep_complete == Some(false))
1536        {
1537            bail!(
1538                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539            );
1540        }
1541        crate::pr_head::ensure_owner(&state)?;
1542        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543            bail!(
1544                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545            );
1546        }
1547        if let Some(to) = &state.released_to {
1548            bail!(
1549                "run {} cannot be resumed: its worktree was released to run {}",
1550                state.short(),
1551                crate::run::short_of(to)
1552            );
1553        }
1554        let roles = state.config.resolve_roles()?;
1555        if let Some(id) = &state.config.graph.selected_implementer
1556            && state.candidates.iter().any(|c| c.agent != *id)
1557        {
1558            bail!(
1559                "replacement implementer pin does not match the saved candidates; task must stay held"
1560            );
1561        }
1562        let max_parallel = state.config.graph.max_parallel.max(1);
1563        Ok(Self {
1564            state,
1565            roles,
1566            sem: Arc::new(Semaphore::new(max_parallel)),
1567            pause: Pause::new(),
1568            interrupt: Pause::new(),
1569        })
1570    }
1571
1572    /// Walk the graph to a terminal state, skipping nodes already recorded.
1573    ///
1574    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1575    /// the phone - ends here, so this is the one place a run that ended
1576    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1577    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1578    pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579        // Allocate before polling: boxing inside an async wrapper would still
1580        // retain its construction frame while the nested graph is polled.
1581        Box::pin(self.execute_inner())
1582    }
1583
1584    async fn execute_inner(&mut self) -> Result<()> {
1585        anyhow::ensure!(
1586            self.state.startup.as_ref().is_none_or(|s| s.complete),
1587            "startup incomplete; inspect retained evidence before explicit recovery"
1588        );
1589        if let Some(selection) = &self.state.pr_recovery
1590            && (selection.source_proof.is_some() || selection.source_run.is_some())
1591        {
1592            Box::pin(selection.verify_source(&self.state.repo)).await?;
1593        }
1594        // Queue claims cover daemon work; this also serializes manual drivers
1595        // consuming a repair answer, including the new review and decision.
1596        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597            || self.state.land_approval.is_some()
1598            || self.state.merge_decision.is_some()
1599            || !self.state.merge_repairs.is_empty()
1600            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601        {
1602            let claim = FixClaim::acquire(&self.state.dir())?;
1603            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604            let disk = RunState::load(&self.state.id)?;
1605            let checkpoint = |s: &RunState| {
1606                (
1607                    s.merge_repairs.len(),
1608                    s.merge_repairs
1609                        .last()
1610                        .map(|r| (r.question.clone(), r.phase)),
1611                    s.land_approval
1612                        .as_ref()
1613                        .map(|a| (a.question.clone(), a.head.clone())),
1614                )
1615            };
1616            if checkpoint(&disk) != checkpoint(&self.state) {
1617                bail!(
1618                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619                );
1620            }
1621            Some(claim)
1622        } else {
1623            None
1624        };
1625        let mut execution = self.claim_driver().await?;
1626        let runner = &mut *execution.runner;
1627        let wait = runner.repository_wait();
1628        // Keep the graph's large future off the callers' stack, including
1629        // standalone task attempts running on a default-sized test thread.
1630        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631        let result = match result {
1632            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633                record_repository_block(&mut runner.state, &e);
1634                runner.park_here()?;
1635                Ok(())
1636            }
1637            other => other,
1638        };
1639        execution.driver.finish(&mut runner.state).await?;
1640        let ended = if result.is_err() {
1641            Some({
1642                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643                if let Err(error) = &result {
1644                    notice.message = format!(
1645                        "Run {} stopped with an error: {}",
1646                        runner.state.id,
1647                        error.to_string().lines().next().unwrap_or("unknown error")
1648                    );
1649                }
1650                notice
1651            })
1652        } else {
1653            crate::notices::run_ended(&runner.state)
1654        };
1655        if let Some(notice) = ended {
1656            crate::notices::raise(notice);
1657        }
1658        result
1659    }
1660
1661    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663    }
1664
1665    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666        let wait = self.repository_wait();
1667        coordinate_operation(&mut self.state, cwd, &wait).await
1668    }
1669
1670    /// Validate ownership and persist the driver claim in one critical section.
1671    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672        let owner = match crate::pr_head::owner_lock_wait(
1673            &crate::run::home(),
1674            &self.state.id,
1675            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676        )
1677        .await
1678        {
1679            Ok(owner) => owner,
1680            Err(error) => {
1681                let detail = format!("worktree maintenance or ownership update: {error:#}");
1682                // Admission did not own run.json: retain this resource outcome
1683                // in memory for queue settlement without overwriting a concurrent
1684                // retirement or publication transaction.
1685                self.state.resource_block = Some(crate::run::ResourceBlock {
1686                    cause: crate::run::ResourceBlockCause::Other,
1687                    node: "start".into(),
1688                    kind: "worktree maintenance".into(),
1689                    detail: detail.clone(),
1690                    at: Timestamp::now(),
1691                });
1692                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693            }
1694        };
1695        let disk = RunState::load(&self.state.id)?;
1696        crate::pr_head::ensure_owner(&disk)?;
1697        let claimed =
1698            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699        // Only proven-live ownership refuses here. Unknown records (legacy runs
1700        // with no claim marker or process identity) fall through to the
1701        // exclusive execution lease taken just below, which is what actually
1702        // keeps a second driver out.
1703        anyhow::ensure!(
1704            disk.liveness(claimed) != crate::run::Liveness::Live,
1705            "run already has a live driver; wait for its execution to end"
1706        );
1707        if let Some(to) = &disk.released_to {
1708            bail!(
1709                "run {} cannot continue: its worktree was released to run {}",
1710                self.state.short(),
1711                crate::run::short_of(to)
1712            );
1713        }
1714        anyhow::ensure!(
1715            serde_json::to_value(&disk.base_sync_recovery)?
1716                == serde_json::to_value(&self.state.base_sync_recovery)?,
1717            "base recovery changed since this run was loaded; reload the saved run"
1718        );
1719        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720        let mut execution = DriverExecution {
1721            runner: self,
1722            driver,
1723        };
1724        // Declare after the execution guard: on any claim error this short
1725        // lock must drop before that guard reloads and releases ownership.
1726        let _owner = owner;
1727        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728            .await?;
1729        let state = &mut execution.runner.state;
1730        // Moving again, so it is no longer parked. Set before the walk rather
1731        // than in `resume`, so every way of re-entering the graph clears it
1732        // and a card cannot claim a run is waiting to be resumed while the
1733        // agents are already working.
1734        state.parked = false;
1735        state.resource_wait = None;
1736        state.resource_block = None;
1737        // Any seat this state still lists as answering belongs to whatever
1738        // process last drove this run — this one included, if it crashed
1739        // mid-wave. Cleared and flushed immediately, before anything else
1740        // runs, so a resume can never show a seat as live when nothing is
1741        // asking it anything yet; the node that actually dispatches the next
1742        // wave repopulates it.
1743        state.clear_active();
1744        // Recorded in the same spot, and flushed together with the clear
1745        // above: this is the pid a reader checks (`RunState::liveness`) when
1746        // no daemon claim exists to answer "is a process still driving this
1747        // run" — a plain `magi run` / `magi review` typed into a terminal
1748        // claims nothing there. Always overwritten, never only-if-absent, so
1749        // a resumed run's stale pid from a previous, possibly-dead process
1750        // can never survive into this one's own report. Unlike
1751        // `clear_active`, this changes on every single `execute()` call, so
1752        // the save below is now unconditional rather than only-if-cleared.
1753        //
1754        // `driver_started_at` is recorded in the same breath, from this same
1755        // pid, so `liveness` can tell a live pid that is genuinely still us
1756        // apart from one the OS has since handed to an unrelated process —
1757        // see that field's own doc for why the pid alone is not enough.
1758        let pid = std::process::id();
1759        state.driver_pid = Some(pid);
1760        state.driver_started_at = crate::proc::process_started_at(pid);
1761        state.driver_exited = false;
1762        state.driver_claimed = Some(true);
1763        state.begin_pass();
1764        execution.driver.bind(state);
1765        state.save()?;
1766        execution.driver.record(state)?;
1767        Ok(execution)
1768    }
1769
1770    /// No conflict checkpoint proves dispatch was never reached. End that
1771    /// consumed generation so an owner can explicitly authorize a fresh one.
1772    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774        if let Err(error) = &result
1775            && !error.is::<crate::repo_lock::ResourceBlocked>()
1776            && self
1777                .state
1778                .repair_grants
1779                .last()
1780                .is_some_and(|g| g.conflict.is_none())
1781        {
1782            self.state.repair_grants.last_mut().unwrap().phase =
1783                crate::repair_budget::Phase::Complete;
1784            self.state.status = RunStatus::Blocked;
1785            self.state.save()?;
1786            bail!(
1787                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788                self.state.passes.last().map_or(0, |p| p.index)
1789            );
1790        }
1791        result
1792    }
1793
1794    /// A grant owns only these repair attempts. Confirmation reviews cannot
1795    /// create another fixer allowance, and every operation is restart-checkpointed.
1796    async fn execute_granted_repair(&mut self) -> Result<()> {
1797        use crate::repair_budget::{Kind, Phase};
1798        let index = self.state.repair_grants.len() - 1;
1799        let winner = self
1800            .state
1801            .winner()
1802            .cloned()
1803            .context("repair winner missing")?;
1804        loop {
1805            let grant = self.state.repair_grants[index].clone();
1806            match grant.phase {
1807                Phase::Pending => {
1808                    if grant.spent >= grant.receipt.request.rounds {
1809                        self.state.repair_grants[index].phase = Phase::Complete;
1810                        self.state.status = RunStatus::Blocked;
1811                        self.state.save()?;
1812                        return Ok(());
1813                    }
1814                    // Reserve before dispatch. A crash can consume an attempt,
1815                    // but can never duplicate it or reset saved fixer history.
1816                    self.state.repair_grants[index].spent += 1;
1817                    self.state.repair_grants[index].phase = Phase::Fixing;
1818                    self.state.save()?;
1819                    match grant.receipt.request.kind {
1820                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821                        Kind::Gate => {
1822                            let failed = self.state.gate.clone();
1823                            let _ = self.gate_fix_round(&winner, &failed).await?;
1824                        }
1825                        Kind::Review => {
1826                            let mut record = self
1827                                .state
1828                                .reviews
1829                                .last()
1830                                .cloned()
1831                                .context("review failure missing")?;
1832                            let round = record.round;
1833                            let step = self.run_fix_round(&winner, &mut record).await?;
1834                            self.state.reviews[round - 1] = record;
1835                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1836                                self.state.repair_grants[index].phase = Phase::Complete;
1837                                self.state.status = RunStatus::Blocked;
1838                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839                                self.state.save()?;
1840                                return Ok(());
1841                            }
1842                            if matches!(step, FixStep::Resource) {
1843                                self.stop_for_resource()?;
1844                                return Ok(());
1845                            }
1846                            self.state.save()?;
1847                        }
1848                    }
1849                    if self.resource_halt()? {
1850                        return Ok(());
1851                    }
1852                }
1853                Phase::Fixing => {
1854                    if grant.receipt.request.kind == Kind::BaseConflict {
1855                        self.repair_base_conflict(false).await?;
1856                    }
1857                    // Interrupted fixers may have left source changes or an
1858                    // unpushed commit. Preserve them before fresh assessment.
1859                    let _coordination = self.coordinate(&winner.worktree).await?;
1860                    let rescued = git::rescue_commit(
1861                        &winner.worktree,
1862                        "magi: preserve interrupted granted repair",
1863                    )
1864                    .await?;
1865                    self.state.note_withheld("repair-grant", &rescued.withheld);
1866                    // Fresh checks must supersede even a same-head failure.
1867                    // Archive rather than erase failure evidence or counters.
1868                    self.state
1869                        .verification_history
1870                        .push(crate::run::VerificationEvidence {
1871                            superseded_by: None,
1872                            archived_at: Some(Timestamp::now()),
1873                            fixes_spent: Some(self.state.gate_fixes.len()),
1874                            gate_head: self.state.gate_head.take(),
1875                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1876                            gate: std::mem::take(&mut self.state.gate),
1877                            e2e: self.state.head_e2e.take(),
1878                        });
1879                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880                        self.state
1881                            .verification_history
1882                            .push(crate::run::VerificationEvidence {
1883                                superseded_by: None,
1884                                archived_at: Some(Timestamp::now()),
1885                                fixes_spent: Some(self.state.gate_fixes.len()),
1886                                gate_head: None,
1887                                gate_ran: false,
1888                                gate: Vec::new(),
1889                                e2e: Some(e2e),
1890                            });
1891                    }
1892                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1893                    self.state.repair_grants[index].head =
1894                        git::rev_parse(&winner.worktree, "HEAD").await?;
1895                    self.state.repair_grants[index].phase = Phase::Reviewing;
1896                    if grant.receipt.request.kind == Kind::BaseConflict {
1897                        self.state.base_sync_recovery.stop = None;
1898                        if let Some(sync) = &mut self.state.base_sync {
1899                            sync.tip = grant
1900                                .receipt
1901                                .request
1902                                .tip
1903                                .clone()
1904                                .context("repair base tip missing")?;
1905                            sync.conflict = None;
1906                            sync.behind = 0;
1907                        }
1908                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909                    }
1910                    self.state.save()?;
1911                }
1912                Phase::Reviewing => {
1913                    self.review_loop_pass().await?;
1914                    if self.resource_halt()? {
1915                        return Ok(());
1916                    }
1917                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918                    let clean = self.state.reviews.len() > grant.review_start
1919                        && self.state.reviews.last().is_some_and(|r| {
1920                            r.head == head
1921                                && r.clean
1922                                && r.expected > 0
1923                                && r.answered == r.expected
1924                                && matches!(
1925                                    r.e2e_status(),
1926                                    E2eStatus::Passed | E2eStatus::NotConfigured
1927                                )
1928                        });
1929                    if !clean {
1930                        // A confirmation review carries no additional fix budget.
1931                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932                            == Kind::Review
1933                            && grant.spent < grant.receipt.request.rounds
1934                            && self.state.reviews.last().is_some_and(|r| {
1935                                r.fix.is_none()
1936                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937                            }) {
1938                            Phase::Pending
1939                        } else {
1940                            Phase::Complete
1941                        };
1942                        self.state.status = RunStatus::Blocked;
1943                        self.state.event(
1944                            "repair-grant",
1945                            "fresh review did not clear the repaired head; no publication",
1946                        );
1947                        self.state.save()?;
1948                        if self.state.repair_grants[index].phase == Phase::Pending {
1949                            continue;
1950                        }
1951                        return Ok(());
1952                    }
1953                    self.state.repair_grants[index].phase = Phase::Gating;
1954                    self.state.save()?;
1955                }
1956                Phase::Gating => {
1957                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958                    self.state.reconcile_verification_head(&head);
1959                    let wait = self.repository_wait();
1960                    if self
1961                        .state
1962                        .reviews
1963                        .last()
1964                        .is_none_or(|r| r.head != head || !r.clean)
1965                    {
1966                        self.state.status = RunStatus::Blocked;
1967                        self.state.repair_grants[index].phase = Phase::Complete;
1968                        self.state.event(
1969                            "repair-grant",
1970                            "head changed since fresh review; repair held",
1971                        );
1972                        self.state.save()?;
1973                        return Ok(());
1974                    }
1975                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976                        return Ok(());
1977                    }
1978                    let outcomes = self.run_gate(&winner).await?;
1979                    if verify_inconclusive(&outcomes) {
1980                        return Ok(());
1981                    }
1982                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983                    if after != head {
1984                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1985                        self.state.repair_grants[index].head = after;
1986                        self.state.repair_grants[index].phase = Phase::Reviewing;
1987                        self.state.status = RunStatus::Blocked;
1988                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989                        self.state.save()?;
1990                        return Ok(());
1991                    }
1992                    let passed = outcomes.iter().all(CommandOutcome::ok);
1993                    // Follow-up grants owe the same guarded checkout delivery
1994                    // as the conflict repair that introduced the new head.
1995                    if passed
1996                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997                    {
1998                        return Ok(());
1999                    }
2000                    self.state.repair_grants[index].succeeded = passed;
2001                    self.state.gate = outcomes;
2002                    self.state.gate_ran = true;
2003                    self.state.status = if passed {
2004                        RunStatus::Gating
2005                    } else {
2006                        RunStatus::Blocked
2007                    };
2008                    self.state.repair_grants[index].phase = if !passed
2009                        && grant.receipt.request.kind == Kind::Gate
2010                        && grant.spent < grant.receipt.request.rounds
2011                    {
2012                        Phase::Pending
2013                    } else {
2014                        Phase::Complete
2015                    };
2016                    if passed && self.state.config.merge.mode == MergeMode::Pr {
2017                        // Persist publication debt with the successful checkpoint.
2018                        self.state.pr_publication_pending = true;
2019                    } else if passed {
2020                        self.state.status = RunStatus::Ready;
2021                    }
2022                    self.state.save()?;
2023                    if !passed {
2024                        continue;
2025                    }
2026                    // Existing PR ownership and push lease stay in place. The
2027                    // normal publisher owes the new reviewed/gated SHA, and
2028                    // land's fresh owner approval remains mandatory.
2029                    if self.state.config.merge.mode != MergeMode::Pr {
2030                        self.state.status = RunStatus::Ready;
2031                        self.state.event(
2032                            "repair-grant",
2033                            "repair verified locally; grant carries no merge authorization",
2034                        );
2035                        self.state.save()?;
2036                        return Ok(());
2037                    }
2038                    return self.merge().await;
2039                }
2040                Phase::Complete => return Ok(()),
2041            }
2042        }
2043    }
2044
2045    async fn execute_graph(&mut self) -> Result<()> {
2046        // Even a terminal run may reenter through an explicit recovery action.
2047        // Restore before prep shortcuts, saved conversations or graph nodes use
2048        // a disposable path; GC may already have removed its original objects.
2049        let archive_home = crate::run::home().canonicalize()?;
2050        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051            let repo = self.state.repo.clone();
2052            let _coordination = self.coordinate(&repo).await?;
2053            ensure_setup_config(&mut self.state, &repo).await?;
2054            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055        }
2056        self.state.configure_recovery_resume()?;
2057        if self
2058            .state
2059            .repair_grants
2060            .last()
2061            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062        {
2063            return self.execute_granted_repair().await;
2064        }
2065        if self
2066            .state
2067            .repair_grants
2068            .last()
2069            .is_some_and(|g| !g.succeeded)
2070        {
2071            self.state.status = RunStatus::Blocked;
2072            self.state.save()?;
2073            return Ok(());
2074        }
2075        // Preservation may invalidate an old empty classification. Resolve it
2076        // before any resume shortcut can reuse a verdict or publish a branch.
2077        self.persist_candidates().await?;
2078        // A saved base stop outranks stage markers, including legacy Voting.
2079        // Recovery never reenters implementation, deliberation or voting.
2080        if self.state.base_sync_blocked() {
2081            self.sync_to_base().await?;
2082            if !self.state.base_sync_blocked()
2083                && self.state.status != RunStatus::AlreadyInBase
2084                && !self.resource_halt()?
2085            {
2086                self.finish_after_tally().await?;
2087            }
2088            return Ok(());
2089        }
2090        if self
2091            .state
2092            .merge_decision
2093            .as_ref()
2094            .is_some_and(|decision| !decision.closed)
2095            && self.state.status != RunStatus::Landing
2096        {
2097            if crate::merge_decision::resumable_hold(&self.state) {
2098                self.state.merge = Some(MergeOutcome {
2099                    mode: MergeMode::Pr,
2100                    ok: false,
2101                    detail: self
2102                        .state
2103                        .merge_decision
2104                        .as_ref()
2105                        .unwrap()
2106                        .target
2107                        .pr
2108                        .clone(),
2109                    empty: false,
2110                });
2111                self.run_land().await?;
2112                self.settle_questions();
2113                return Ok(());
2114            }
2115            bail!(
2116                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117            );
2118        }
2119        if self.state.config.graph.selected_implementer.is_some()
2120            && self.state.status == RunStatus::Blocked
2121            && self.state.tally.is_none()
2122            && self.state.candidate_preservation.is_empty()
2123            && self.state.interrupted_implementations.is_empty()
2124        {
2125            bail!(
2126                "selected implementation is held; explicitly release the owning task to retry it"
2127            );
2128        }
2129
2130        if self.state.merge_repairs.last().is_some_and(|r| {
2131            matches!(
2132                r.phase,
2133                crate::run::MergeRepairPhase::Requested
2134                    | crate::run::MergeRepairPhase::Fixing
2135                    | crate::run::MergeRepairPhase::Reviewing
2136            )
2137        }) {
2138            if self.repair_merge_review().await? {
2139                self.run_land().await?;
2140            }
2141            self.settle_questions();
2142            return Ok(());
2143        }
2144        // Publication already passed review and gate. Resume only that node:
2145        // syncing to a newer base here could change the verified commit.
2146        if self.state.pr_publication_pending {
2147            self.merge().await?;
2148            if self.resource_halt()? {
2149                return Ok(());
2150            }
2151            self.state.save()?;
2152            self.settle_questions();
2153            return Ok(());
2154        }
2155        // A run that already lost its quorum never resumes into the verdict
2156        // machinery: `deliberate` and `vote` would otherwise clobber the
2157        // stalled marker back to Voting and the run would keep going past a
2158        // verdict that is no longer trustworthy. Everything already recorded is
2159        // kept, so the run stays resumable (or foldable) for a human to pick up.
2160        //
2161        // On --resume the run gets one chance to repair itself: the seats a
2162        // rate limit took out are re-asked. If their quota has since reset and
2163        // the quorum is restored, the run picks up and finishes; otherwise it
2164        // stays stale and still-resumable for a later retry. If it does not
2165        // recover, the returned status stays `Stalled` and nothing was
2166        // clobbered (the recovery only mutates entries for the lost seats).
2167        if self.state.status == RunStatus::Stalled {
2168            if self.recover_stall().await? {
2169                self.finish_after_tally().await?;
2170            } else {
2171                // Still below quorum: persist the marker and stay resumable.
2172                self.state.save()?;
2173            }
2174            return Ok(());
2175        }
2176        // A run parked inside `land` - watching CI, mid fix-round, or
2177        // waiting on the owner's merge approval - resumes directly into it,
2178        // never back through `prep`. Everything before `merge` already
2179        // concluded; that is the only way `status` reaches `Landing` in the
2180        // first place. Re-walking `review_loop` first would also be actively
2181        // wrong: its own status recomputation (see its doc) treats any
2182        // clean round as reason to set `status` to `Gating`, which would
2183        // clobber this marker before `merge` ever ran, and this run would
2184        // never find its way back into `land` at all.
2185        if self.state.status == RunStatus::Landing
2186            || crate::handover::landing_checkpoint(&self.state)
2187        {
2188            self.state.land_sync_pending = true;
2189            self.state.save()?;
2190            self.run_land().await?;
2191            // `run_land` may have settled the run right here - CI came back
2192            // green and the PR merged, say - without ever passing back
2193            // through `merge`'s own trailing call. Whatever it left `status`
2194            // as is what this has to read.
2195            self.settle_questions();
2196            return Ok(());
2197        }
2198        // A run parked at `merge` on the owner's word about a withheld PR text
2199        // resumes straight into `merge`. Re-walking the tail would sync to the
2200        // base again (possibly rebasing, possibly spending the fixer) after the
2201        // gate had passed, and then push a tree nobody reviewed or gated.
2202        if self.state.github_text.is_some()
2203            && self.state.merge.is_none()
2204            && self.state.status == RunStatus::Gating
2205        {
2206            self.merge().await?;
2207            if self.resource_halt()? {
2208                return Ok(());
2209            }
2210            self.state.save()?;
2211            return Ok(());
2212        }
2213        if let Err(e) = self.prep().await {
2214            if e.is::<crate::repo_lock::ResourceBlocked>() {
2215                return Err(e);
2216            }
2217            self.state.status = RunStatus::Blocked;
2218            self.state
2219                .prep_failure
2220                .get_or_insert_with(|| format!("{e:#}"));
2221            self.state
2222                .event("prep", format!("preparation stopped: {e:#}"));
2223            self.state.save()?;
2224            return Err(e);
2225        }
2226        if self.park_here()? {
2227            return Ok(());
2228        }
2229        self.advise().await?;
2230        if self.park_here()? {
2231            return Ok(());
2232        }
2233        self.implement().await?;
2234        if self.park_here()? {
2235            return Ok(());
2236        }
2237        // `after_implement` already saved the state and settled any open
2238        // questions when it set this; nothing later in the graph has
2239        // anything to judge.
2240        if self.state.status == RunStatus::VerifiedNoop {
2241            return Ok(());
2242        }
2243        self.judge().await?;
2244        if self.park_here()? {
2245            return Ok(());
2246        }
2247        self.deliberate().await?;
2248        if self.park_here()? {
2249            return Ok(());
2250        }
2251        self.vote().await?;
2252        if self.park_here()? {
2253            return Ok(());
2254        }
2255        self.tally()?;
2256        // A verdict that lost its quorum is not trustworthy: do not review,
2257        // gate, or merge on it. Everything already done is kept, so the run
2258        // stays resumable (or foldable); the human can replace the agent that
2259        // ran out of quota and pick it up.
2260        if self.state.status == RunStatus::Stalled {
2261            // Persist the stalled marker now — the normal end-of-execute save
2262            // below is below this early return, and without it a resumed run
2263            // would reload a pre-tally status and keep going.
2264            self.state.save()?;
2265            return Ok(());
2266        }
2267        self.finish_after_tally().await?;
2268        Ok(())
2269    }
2270
2271    /// A wave could not start its agents because a shared resource stayed busy
2272    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2273    /// resumable. Unstarted seats have no result; completed peers retain
2274    /// their saved candidate state.
2275    fn stop_for_resource(&mut self) -> Result<()> {
2276        // Resource acquisition observes cancellation inside a node. Complete
2277        // the same durable park as every other node boundary before returning.
2278        if self.pause.parked() || self.interrupt.parked() {
2279            self.park_here()?;
2280            return Ok(());
2281        }
2282        if self.state.resource_block.is_some() {
2283            self.state.status = RunStatus::Blocked;
2284        }
2285        self.state.save()
2286    }
2287
2288    /// Stop walking the graph if a node ended on a resource block.
2289    fn resource_halt(&mut self) -> Result<bool> {
2290        if self.state.resource_block.is_none() {
2291            return Ok(false);
2292        }
2293        self.stop_for_resource()?;
2294        Ok(true)
2295    }
2296
2297    /// Park here if asked to, recording it in the run's own timeline.
2298    ///
2299    /// Returns whether the caller should stop walking the graph. The state is
2300    /// saved either way by the node that just finished; this adds the event so
2301    /// the operator's card says why a run that is neither finished nor moving
2302    /// is sitting where it is.
2303    fn park_here(&mut self) -> Result<bool> {
2304        let parking = self.pause.parked() || self.interrupt.parked();
2305        if !parking && self.state.resource_block.is_some() {
2306            self.stop_for_resource()?;
2307            return Ok(true);
2308        }
2309        // Either handle asking is enough - see `Pause`'s own doc for why
2310        // they are never the same one. `interrupt` is checked second so a
2311        // reason it carries is preferred in the message below over a plain
2312        // shutdown park racing it at the same boundary.
2313        if !self.pause.parked() && !self.interrupt.parked() {
2314            return Ok(false);
2315        }
2316        if self.state.parked {
2317            return Ok(true);
2318        }
2319        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320        if let Some(block) = &mut self.state.resource_block
2321            && matches!(
2322                block.cause,
2323                crate::run::ResourceBlockCause::Cancelled { .. }
2324            )
2325        {
2326            block.cause = crate::run::ResourceBlockCause::Cancelled {
2327                park: cause.clone(),
2328            };
2329        }
2330        let why = match cause.as_ref().map(ParkCause::label) {
2331            Some(reason) => format!(
2332                "parked after `{}` ({reason}) — resume to carry on from here",
2333                self.state.status.as_str()
2334            ),
2335            None => format!(
2336                "parked after `{}` — resume to carry on from here",
2337                self.state.status.as_str()
2338            ),
2339        };
2340        self.state.event("park", why);
2341        self.state
2342            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343        self.state.parked = true;
2344        self.state.save()?;
2345        Ok(true)
2346    }
2347
2348    /// Only a new explicit operator approval may retry failed preparation.
2349    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350        if self.state.prep_failure.is_none() {
2351            return Ok(());
2352        }
2353        let failed_at = self
2354            .state
2355            .events
2356            .iter()
2357            .rev()
2358            .find(|event| event.node == "prep")
2359            .map_or(self.state.updated_at, |event| event.at);
2360        anyhow::ensure!(
2361            approval_at > failed_at,
2362            "preparation retry requires a new explicit release after the failure"
2363        );
2364        self.state.prep_failure = None;
2365        self.state.prep_complete = Some(false);
2366        self.state
2367            .event("prep-retry", "operator approved retry of preparation");
2368        Ok(())
2369    }
2370
2371    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2372    pub fn on_pause(&mut self, pause: Pause) {
2373        self.pause = pause;
2374    }
2375
2376    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2377    /// scheduler asking this one run - and no other - to park so a task
2378    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2379    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2380    /// handle.
2381    pub fn watch_interrupt(&mut self, pause: Pause) {
2382        self.interrupt = pause;
2383    }
2384
2385    /// Abandon this run's own open questions, once `status` has actually
2386    /// settled rather than merely paused.
2387    ///
2388    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2389    /// either back up with the candidates, the review round and the seat
2390    /// sessions already on disk, so a question an implementer asked mid-round
2391    /// may still get a real answer read by a real resume. Only the statuses
2392    /// `resumable` excludes are actually final: the run merged, it reached
2393    /// `Ready` with nothing left to do, it failed outright with no
2394    /// established point to continue from, or every candidate agreed, with
2395    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2396    /// every one of those the seat that asked is gone for good, exactly like
2397    /// the run being deleted under `magi run rm` - so the same cleanup
2398    /// applies, worded for what actually happened instead of "the run was
2399    /// deleted".
2400    ///
2401    /// Best-effort and silent on success: called from every place `status`
2402    /// can land on one of those three, including ones a resumed run revisits,
2403    /// so it must cost nothing when there was nothing open to begin with.
2404    fn settle_questions(&mut self) {
2405        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407        }
2408    }
2409
2410    /// The tail of the graph after a trustworthy tally: fold losers, review,
2411    /// gate, merge, and persist.
2412    fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413        Box::pin(self.finish_after_tally_inner())
2414    }
2415
2416    async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417        // Before the losers are folded: their branches are what the graft
2418        // reads from.
2419        self.graft().await?;
2420        if self.resource_halt()? {
2421            return Ok(());
2422        }
2423        self.fold_losers().await?;
2424        // Before review starts, and again right before the gate: a run's
2425        // review rounds can themselves take long enough for the base to move
2426        // a second time, and the gate is the one node whose "green" gets
2427        // acted on.
2428        self.sync_to_base().await?;
2429        if self.hold_base_sync().await? || self.resource_halt()? {
2430            return Ok(());
2431        }
2432        if self.state.status == RunStatus::AlreadyInBase {
2433            return Ok(());
2434        }
2435        self.review_loop().await?;
2436        if self.resource_halt()? {
2437            return Ok(());
2438        }
2439        self.sync_to_base().await?;
2440        if self.hold_base_sync().await? || self.resource_halt()? {
2441            return Ok(());
2442        }
2443        if self.state.status == RunStatus::AlreadyInBase {
2444            return Ok(());
2445        }
2446        self.gate().await?;
2447        self.merge().await?;
2448        if self.resource_halt()? {
2449            return Ok(());
2450        }
2451        self.state.save()?;
2452        Ok(())
2453    }
2454
2455    // ---------------------------------------------------------------- prep
2456
2457    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2458    /// run with the step and its output in the event; it is never swallowed.
2459    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2460    /// resume starts over instead of finding candidates and missing seats.
2461    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462        let result = match ensure_setup_config(&mut self.state, repo).await {
2463            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464            Err(e) => Err(e),
2465        };
2466        if let Err(e) = result {
2467            git::worktree_remove_clean(repo, wt).await.ok();
2468            if in_prep {
2469                self.rollback_prep(repo).await;
2470            }
2471            self.state.status = RunStatus::Blocked;
2472            self.state.event(
2473                "setup",
2474                format!("worktree setup failed in {}: {e:#}", wt.display()),
2475            );
2476            self.state.save()?;
2477            return Err(e);
2478        }
2479        Ok(())
2480    }
2481
2482    /// Undo everything `prep` created, so a resume re-runs it whole.
2483    async fn rollback_prep(&mut self, repo: &Path) {
2484        let root = self.state.worktree_root();
2485        for c in std::mem::take(&mut self.state.candidates) {
2486            if matches!(
2487                git::worktree_remove_clean(repo, &c.worktree).await,
2488                Ok(true)
2489            ) {
2490                git::branch_delete(repo, &c.branch).await.ok();
2491            } else {
2492                self.state.candidates.push(c);
2493            }
2494        }
2495        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496        for j in 1..=seats {
2497            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2498                .await
2499                .ok();
2500        }
2501        for k in 1..=self.state.config.graph.advisors {
2502            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2503                .await
2504                .ok();
2505        }
2506    }
2507
2508    async fn prep(&mut self) -> Result<()> {
2509        if let Some(reason) = &self.state.prep_failure {
2510            bail!(
2511                "candidate preparation previously failed: {reason}; correct the source before retrying"
2512            );
2513        }
2514        if self
2515            .state
2516            .prep_complete
2517            .unwrap_or(!self.state.candidates.is_empty())
2518        {
2519            return Ok(());
2520        }
2521        let repo = self.state.repo.clone();
2522        // Includes rollback, config, refs, checkout creation and setup. No
2523        // repository mutation may precede the real shared lock acquisition.
2524        let _coordination = self.coordinate(&repo).await?;
2525        let base = self.state.base_commit.clone();
2526        let plan = self.state.sources.plan(&repo, &base).await?;
2527        self.rollback_prep(&repo).await;
2528        if !self.state.candidates.is_empty() {
2529            bail!(
2530                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531            );
2532        }
2533        self.state.prep_complete = Some(false);
2534        self.state.status = RunStatus::Prep;
2535        self.state.save()?;
2536        let repo = self.state.repo.clone();
2537        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538        let root = self.state.worktree_root();
2539        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541        // The hook is the write-time half of the blindness contract; the
2542        // presentation filter in `blind` is the half that cannot be bypassed.
2543        let hooks_dir = self.state.dir().join("hooks");
2544        if self.state.config.blind.commit_msg_hook {
2545            std::fs::create_dir_all(&hooks_dir)
2546                .with_context(|| format!("create {}", hooks_dir.display()))?;
2547            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548            let path = hooks_dir.join("commit-msg");
2549            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550            make_executable(&path)?;
2551            // Ref-counted rather than a plain idempotent set: with more than
2552            // one run able to be in flight in the same repository at once
2553            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2554            // true?" check cannot tell "another run of mine still needs
2555            // this" from "nobody does", and the run that happens to finish
2556            // first would disable the hook out from under a sibling still
2557            // relying on it.
2558            git::acquire_worktree_config(&repo).await?;
2559            self.state.enabled_worktree_config = true;
2560        }
2561
2562        // `[worktree] setup` hides its products per worktree, which needs the
2563        // same per-worktree config the hook does. Held until fold, like it.
2564        ensure_setup_config(&mut self.state, &repo).await?;
2565
2566        for (index, (spec, label)) in self
2567            .roles
2568            .implementers
2569            .clone()
2570            .into_iter()
2571            .zip(labels)
2572            .enumerate()
2573        {
2574            let branch = self.state.branch_for(label);
2575            let worktree = root.join(format!("cand-{label}"));
2576            // Persist intent before Git can leave a partially created tree.
2577            // Candidate registration follows setup, so it cannot inventory
2578            // crashes or refusals during creation itself.
2579            self.state.event(
2580                "prep-intent",
2581                serde_json::json!({
2582                    "branch": branch,
2583                    "worktree": worktree,
2584                    "start": start,
2585                    "picks": plan.picks,
2586                })
2587                .to_string(),
2588            );
2589            self.state.save()?;
2590            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591            if self.state.config.blind.commit_msg_hook {
2592                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593            }
2594            git::local_exclude(&worktree, "/.magi/").await?;
2595            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596                // Nothing of this prep may survive: a resume must start over,
2597                // not find half the candidates and skip the rest.
2598                git::worktree_remove_clean(&repo, &worktree).await.ok();
2599                git::branch_delete(&repo, &branch).await.ok();
2600                self.rollback_prep(&repo).await;
2601                self.state.status = RunStatus::Blocked;
2602                self.state
2603                    .event("prep", format!("worktree setup failed: {e:#}"));
2604                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605                self.state.save()?;
2606                return Err(e);
2607            }
2608            self.state.candidates.push(Candidate {
2609                index,
2610                label,
2611                agent: spec.id.clone(),
2612                branch,
2613                worktree,
2614                summary: String::new(),
2615                stat: String::new(),
2616                files: 0,
2617                commits: 0,
2618                empty: false,
2619                failed: None,
2620                verified_noop: None,
2621                duration_ms: 0,
2622                folded: false,
2623            });
2624            self.state.save()?;
2625            for pick in &plan.picks {
2626                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627                {
2628                    self.state.status = RunStatus::Blocked;
2629                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630                    self.state
2631                        .event("prep", format!("cannot apply referenced commit: {e}"));
2632                    self.state.save()?;
2633                    return Err(e);
2634                }
2635            }
2636        }
2637
2638        for j in 1..=self.roles.judges.len() {
2639            let wt = root.join(format!("judge-{j}"));
2640            if !wt.exists() {
2641                git::worktree_add_detached(&repo, &wt, &base).await?;
2642                self.setup_seat_worktree(&repo, &wt, true).await?;
2643            }
2644        }
2645
2646        // Disposable, detached checkouts for the design-deliberation stage's
2647        // advisor seats — the same shape as the judges' above, at the same
2648        // base commit, since advisors also only ever read. Sized off the
2649        // configured count directly rather than a resolved roster: unlike
2650        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2651        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2652        // `prep` has no `ResolvedRoles` field to read a count from here.
2653        if self.state.config.graph.advise {
2654            for k in 1..=self.state.config.graph.advisors {
2655                let wt = root.join(format!("advisor-{k}"));
2656                if !wt.exists() {
2657                    git::worktree_add_detached(&repo, &wt, &base).await?;
2658                    self.setup_seat_worktree(&repo, &wt, true).await?;
2659                }
2660            }
2661        }
2662
2663        // A judge cannot tell it is looking at its own patch — the seats keep
2664        // separate conversations — but a panel that shares agents with the
2665        // field is less independent than it looks, and that is worth saying out
2666        // loud once per run rather than leaving it in the config.
2667        let authors: Vec<&str> = self
2668            .roles
2669            .implementers
2670            .iter()
2671            .map(|a| a.id.as_str())
2672            .collect();
2673        let overlap: Vec<String> = self
2674            .roles
2675            .judges
2676            .iter()
2677            .enumerate()
2678            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680            .collect();
2681        if !overlap.is_empty() {
2682            let note = format!(
2683                "{} also authored a candidate; blind, but the panel is less \
2684                 independent than {} distinct agents would be",
2685                overlap.join(", "),
2686                self.roles.judges.len()
2687            );
2688            self.state.event("prep", note);
2689        }
2690
2691        self.state.event(
2692            "prep",
2693            format!(
2694                "{} candidates, {} judges, base {} ({})",
2695                self.state.candidates.len(),
2696                self.roles.judges.len(),
2697                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698                self.state.base_branch
2699            ),
2700        );
2701        self.state.prep_complete = Some(true);
2702        self.state.status = RunStatus::Implementing;
2703        self.state.save()?;
2704        Ok(())
2705    }
2706
2707    // -------------------------------------------------------------- advise
2708
2709    /// The design-deliberation stage: independent, read-only advisor seats
2710    /// each sketch a design before any implementer touches the repository,
2711    /// and (when at least one produced a usable proposal) a synthesis seat
2712    /// blends them into a brief `implement` carries in every candidate's
2713    /// prompt.
2714    ///
2715    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2716    /// advisors` is the proposal count. Everything here is best-effort and
2717    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2718    /// that cannot reach quota, or a synthesis seat that produced nothing
2719    /// usable all leave `implement` exactly as it was before this stage
2720    /// existed — the task instruction alone — rather than failing the whole
2721    /// competition over an enrichment stage. Every outcome is still recorded
2722    /// as an event, so a run that got nothing from this stage says why.
2723    ///
2724    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2725    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2726    /// resumed run whose stage failed would re-run it, and re-spend the
2727    /// agent calls, on every reentry before `implement`.
2728    ///
2729    /// Also skipped once any candidate shows implementation progress — the
2730    /// exact predicate `implement` itself uses to decide a candidate is no
2731    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2732    /// is not enough: a run created by an older binary that predates this
2733    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2734    /// an already-`Implementing`-or-later run under this build would
2735    /// otherwise walk straight back through `prep` (a no-op once candidates
2736    /// exist) into this node and spawn every advisor seat against worktrees
2737    /// `prep` never recreated — after implementation has already started,
2738    /// which is exactly the invariant this stage exists to guarantee.
2739    async fn advise(&mut self) -> Result<()> {
2740        let implement_untouched = self
2741            .state
2742            .candidates
2743            .iter()
2744            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745        if !self.state.config.graph.advise || self.state.advise_attempted {
2746            return Ok(());
2747        }
2748        if !implement_untouched {
2749            self.state.event(
2750                "advise",
2751                "skipping the design-deliberation stage: at least one \
2752                 candidate already shows implementation progress, so this \
2753                 run is past the point the stage exists to run before"
2754                    .to_owned(),
2755            );
2756            self.state.advise_attempted = true;
2757            self.state.save()?;
2758            return Ok(());
2759        }
2760        let run_id = self.state.id.clone();
2761        let prompts = self.state.config.prompts.clone();
2762        let instruction = self.state.instruction.clone();
2763        let language = self.state.config.graph.language.clone();
2764        let root = self.state.worktree_root();
2765        let n = self.state.config.graph.advisors;
2766        let where_recorded = self.state.dir().join("run.json");
2767
2768        let seats = match self.state.config.advisors() {
2769            Ok(seats) if !seats.is_empty() => seats,
2770            Ok(_) => {
2771                self.state.event(
2772                    "advise",
2773                    format!(
2774                        "[graph] advisors is 0; skipping the design-deliberation \
2775                         stage and continuing without a synthesis brief (see {})",
2776                        where_recorded.display()
2777                    ),
2778                );
2779                self.state.advise_attempted = true;
2780                self.state.save()?;
2781                return Ok(());
2782            }
2783            Err(e) => {
2784                self.state.event(
2785                    "advise",
2786                    format!(
2787                        "could not resolve advisor seats ({e:#}); continuing \
2788                         without a design-deliberation brief (see {})",
2789                        where_recorded.display()
2790                    ),
2791                );
2792                self.state.advise_attempted = true;
2793                self.state.save()?;
2794                return Ok(());
2795            }
2796        };
2797
2798        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799        let artifacts = agent::artifacts_dir(&self.state.dir());
2800        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802        let mut jobs = Vec::new();
2803        for (i, spec) in seats.iter().cloned().enumerate() {
2804            let seat_key = format!("advisor-{}", i + 1);
2805            let seat = self.seat(&seat_key, &spec.id);
2806            jobs.push(SeatJob {
2807                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808                spec,
2809                seat,
2810                cwd: worktrees[i % worktrees.len()].clone(),
2811                timeout,
2812                allow_write: false,
2813                sessions: false,
2814                artifacts: artifacts.clone(),
2815                stem: seat_key,
2816                handover: None,
2817            });
2818        }
2819
2820        self.state.event(
2821            "advise",
2822            format!(
2823                "{} advisor seat(s) sketching a design in parallel",
2824                jobs.len()
2825            ),
2826        );
2827        let mut quota_losses = Vec::new();
2828        let cache = self.state.config.cache_dir();
2829        let ctx = WaveCtx {
2830            carry_seats: false,
2831            pauses: Some((&self.pause, &self.interrupt)),
2832            run: &run_id,
2833            node: "advise",
2834            prompts: &prompts,
2835            cache: cache.as_deref(),
2836            round: None,
2837        };
2838        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839        let results = ask_json_wave::<Proposal>(
2840            jobs,
2841            Arc::clone(&self.sem),
2842            self.state.config.graph.retries,
2843            &advisor_roster,
2844            &ctx,
2845            &mut quota_losses,
2846            &mut self.state,
2847            &|p: &Proposal| p.validate(),
2848        )
2849        .await;
2850        if self.state.resource_block.is_some() {
2851            return self.stop_for_resource();
2852        }
2853        self.state.quota.extend(quota_losses);
2854
2855        let mut records = Vec::with_capacity(results.len());
2856        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857            let agent_id = seat.agent.clone();
2858            self.state.seats.insert(seat.key.clone(), seat);
2859            match res {
2860                Ok((proposal, out)) => {
2861                    self.state
2862                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2863                    records.push(advise::AdvisorRecord::proposed(
2864                        i + 1,
2865                        agent_id,
2866                        proposal,
2867                        out.duration_ms,
2868                    ));
2869                }
2870                Err(e) => {
2871                    self.state.event(
2872                        "advise",
2873                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874                    );
2875                    records.push(advise::AdvisorRecord::failed(
2876                        i + 1,
2877                        agent_id,
2878                        e.to_string(),
2879                    ));
2880                }
2881            }
2882        }
2883
2884        let mut advice = advise::Advice {
2885            records,
2886            synthesis: None,
2887        };
2888        if advice.proposals().is_empty() {
2889            self.state.event(
2890                "advise",
2891                "no advisor produced a usable proposal; continuing without a \
2892                 synthesis brief"
2893                    .to_owned(),
2894            );
2895        } else {
2896            match self
2897                .synthesize_brief(
2898                    &advice,
2899                    &instruction,
2900                    &language,
2901                    &worktrees[0],
2902                    &artifacts,
2903                    &run_id,
2904                    &prompts,
2905                    cache.as_deref(),
2906                )
2907                .await
2908            {
2909                Ok(Some(text)) => {
2910                    self.state.event(
2911                        "advise",
2912                        "synthesized a design brief for the implementer".to_owned(),
2913                    );
2914                    advice.synthesis = Some(text);
2915                }
2916                Ok(None) => {
2917                    self.state.event(
2918                        "advise",
2919                        "the synthesis seat produced nothing usable; continuing \
2920                         without a design brief"
2921                            .to_owned(),
2922                    );
2923                }
2924                Err(e) => {
2925                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2926                        return Err(e);
2927                    }
2928                    self.state.event(
2929                        "advise",
2930                        format!("could not synthesize a design brief: {e:#}"),
2931                    );
2932                }
2933            }
2934        }
2935        advise::apply_reflection(&mut advice);
2936
2937        self.state.advice = Some(advice);
2938        self.state.advise_attempted = true;
2939        self.state.save()?;
2940        Ok(())
2941    }
2942
2943    /// The synthesis seat: reads every advisor's proposal and blends them
2944    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2945    /// out of [`Runner::advise`] only for readability — it is not called
2946    /// anywhere else.
2947    ///
2948    /// Picked the same way [`crate::talk`]'s standing conversation and
2949    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2950    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2951    /// order (a claude seat, else the first runnable agent in roster order)
2952    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2953    /// in [`crate::config`] for why a dedicated field exists here at all.
2954    #[allow(clippy::too_many_arguments)]
2955    async fn synthesize_brief(
2956        &mut self,
2957        advice: &advise::Advice,
2958        instruction: &str,
2959        language: &str,
2960        cwd: &Path,
2961        artifacts: &Path,
2962        run_id: &str,
2963        prompts: &Prompts,
2964        cache: Option<&Path>,
2965    ) -> Result<Option<String>> {
2966        let chain = agent::pick_chain(
2967            &self.state.config.agents,
2968            self.state.config.roles.synthesizer.as_ref(),
2969            &agent::installed,
2970            "synthesizer",
2971        )?;
2972        let proposals = advice.proposals();
2973        let mut prompt = prompt::with_overlay(
2974            prompt::synthesize_brief(instruction, &proposals, language),
2975            prompts.overlay("advise"),
2976        );
2977        if cache.is_some() {
2978            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2979            // below — see `prompt::build_cache_note`'s doc for why telling a
2980            // read-only seat to build through the shared cache is exactly how
2981            // a sandbox's write refusal gets misread as a defect.
2982            prompt.push('\n');
2983            prompt.push_str(&prompt::build_cache_note("advise", false));
2984        }
2985        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986        // Each id is tried once, in order; a quota hit, error or unusable
2987        // answer moves to the next. The seat is single-turn (`sessions:
2988        // false`) and the prompt is the whole context, so a fallback agent
2989        // needs nothing carried over.
2990        let mut last = None;
2991        for (n, spec) in chain.iter().enumerate() {
2992            if n > 0 {
2993                self.state
2994                    .event("advise", format!("synthesis falling back to {}", spec.id));
2995            }
2996            let mut seat = self.seat("advise-synthesis", &spec.id);
2997            let wait = self.repository_wait();
2998            let outcome = wait
2999                .scope(agent::invoke(
3000                    spec,
3001                    &mut seat,
3002                    &Invocation {
3003                        coordination: self.state.config.coordination.as_ref(),
3004                        fetched_base: None,
3005                        cwd,
3006                        prompt: &prompt,
3007                        timeout,
3008                        allow_write: false,
3009                        sessions: false,
3010                        artifacts,
3011                        stem: &if n == 0 {
3012                            "advise-synthesis".to_owned()
3013                        } else {
3014                            format!("advise-synthesis-{}", spec.id)
3015                        },
3016                        run: run_id,
3017                        node: "advise",
3018                        cache_dir: None,
3019                        attachments: &[],
3020                        writable: &[],
3021                    },
3022                ))
3023                .await;
3024            if outcome.is_ok() {
3025                self.state.seats.insert(seat.key.clone(), seat);
3026            }
3027            let advance = agent::chain_advances(&outcome);
3028            last = Some(outcome);
3029            if !advance {
3030                break;
3031            }
3032        }
3033        // Exhausted: the last attempt's result is what a single failed seat
3034        // would have produced.
3035        let out = last.expect("a chain holds at least one agent")?;
3036        if !out.usable() {
3037            return Ok(None);
3038        }
3039        let text =
3040            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041        Ok((!text.trim().is_empty()).then_some(text))
3042    }
3043
3044    // ----------------------------------------------------------- implement
3045
3046    async fn implement(&mut self) -> Result<()> {
3047        self.persist_candidates().await?;
3048        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3049        // agent files with `magi task add` name the run that paid for it. The
3050        // prompt overlay is cloned alongside it because the waves borrow it
3051        // while `self` is mutably borrowed by the node's own bookkeeping.
3052        let run_id = self.state.id.clone();
3053        let prompts = self.state.config.prompts.clone();
3054        let todo: Vec<usize> = self
3055            .state
3056            .candidates
3057            .iter()
3058            .enumerate()
3059            .filter(|(_, c)| {
3060                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061                    && c.failed.is_none()
3062                    && !c.empty
3063            })
3064            .map(|(i, _)| i)
3065            .collect();
3066        if todo.is_empty() {
3067            return self.after_implement();
3068        }
3069        self.state.status = RunStatus::Implementing;
3070
3071        let language = self.state.config.graph.language.clone();
3072        let github_language = self.state.config.graph.github_language.clone();
3073        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074        let sessions = self.state.config.graph.sessions;
3075        let artifacts = agent::artifacts_dir(&self.state.dir());
3076        // The design-deliberation stage's blended brief, when `advise` found
3077        // one — carried into every implementer's prompt the same way
3078        // regardless of which candidate it is.
3079        let brief = self
3080            .state
3081            .advice
3082            .as_ref()
3083            .and_then(|a| a.synthesis.as_deref())
3084            .map(str::to_owned);
3085        let attachments = self.state.attachments.clone();
3086
3087        let mut jobs = Vec::new();
3088        for &i in &todo {
3089            let (index, label, worktree) = {
3090                let c = &self.state.candidates[i];
3091                (c.index, c.label, c.worktree.clone())
3092            };
3093            let spec = self.roles.implementers[index].clone();
3094            let seat_key = format!("impl-{label}");
3095            let seat = self.seat(&seat_key, &spec.id);
3096            let instruction = seeded_instruction(&self.state);
3097            jobs.push(SeatJob {
3098                spec,
3099                seat,
3100                prompt: prompt::implement_in(
3101                    &instruction,
3102                    &worktree.to_string_lossy(),
3103                    &language,
3104                    &github_language,
3105                    brief.as_deref(),
3106                    &attachments,
3107                ),
3108                cwd: worktree,
3109                timeout,
3110                allow_write: true,
3111                sessions,
3112                artifacts: artifacts.clone(),
3113                stem: format!("impl-{label}"),
3114                handover: None,
3115            });
3116        }
3117
3118        self.state.event(
3119            "implement",
3120            format!("{} candidates in parallel", jobs.len()),
3121        );
3122        // Kept so a seat whose CLI hung up can be asked again from the same
3123        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3124        // can update a seat's own entry once a fallback agent takes it over —
3125        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3126        // whichever agent actually answered, not the one that quota'd out.
3127        let mut sent = jobs.clone();
3128        let cache = self.state.config.cache_dir();
3129        let ctx = WaveCtx {
3130            carry_seats: false,
3131            pauses: Some((&self.pause, &self.interrupt)),
3132            run: &run_id,
3133            node: "implement",
3134            prompts: &prompts,
3135            cache: cache.as_deref(),
3136            round: None,
3137        };
3138        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139        if !wave_blocked(&results) {
3140            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141                .await;
3142            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143                .await?;
3144            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145                .await;
3146        }
3147        // Do not finalize failed candidates when the entire wave needs machine
3148        // recovery: leaving them pending resumes this node with its work intact.
3149        if !results.is_empty()
3150            && results
3151                .iter()
3152                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153        {
3154            let losses: Vec<_> = results
3155                .iter()
3156                .map(|(_, seat, out)| {
3157                    let AgentOutcome::Quota(o) = out else {
3158                        unreachable!()
3159                    };
3160                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161                })
3162                .collect();
3163            self.state.quota.extend(losses.clone());
3164            for (_, seat, _) in results {
3165                self.state.seats.insert(seat.key.clone(), seat);
3166            }
3167            for &i in &todo {
3168                let label = self.state.candidates[i].label;
3169                self.state.interrupted_implementations.insert(label);
3170                self.state
3171                    .candidate_preservation
3172                    .insert(label, "machine outage awaiting persistence".into());
3173            }
3174            // Preserve partial edits too, while keeping every interrupted seat
3175            // pending even when its tree already contains an inherited patch.
3176            block_machine_outage(&mut self.state, "implement", &losses);
3177            self.state.save()?;
3178            self.persist_candidates().await?;
3179            for &i in &todo {
3180                self.state
3181                    .interrupted_implementations
3182                    .insert(self.state.candidates[i].label);
3183            }
3184            return self.stop_for_resource();
3185        }
3186        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189            && results
3190                .iter()
3191                .any(|(_, _, out)| FailClass::of(out).is_some());
3192        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194                // Executed turns interrupted by disk pressure already carry
3195                // a preservation barrier from wave. Admission-only waits do
3196                // not. Keep the interrupted seat's session for recovery too.
3197                if self
3198                    .state
3199                    .candidate_preservation
3200                    .contains_key(&self.state.candidates[i].label)
3201                {
3202                    self.state.seats.insert(seat.key.clone(), seat);
3203                }
3204                continue;
3205            }
3206            let seat_key = seat.key.clone();
3207            // A quota fallback (`resume_seat_handovers`) may have handed this
3208            // seat to a different agent than the one `prep` recorded on the
3209            // candidate; the stats tables and any later fixer-defaults-to-
3210            // winner's-author lookup must credit whoever actually answered —
3211            // unless every fallback also quota'd out, in which case nobody
3212            // actually answered and crediting the last agent tried would
3213            // erase every earlier agent's own quota loss from the stats
3214            // tables instead of just this one seat's.
3215            let agent = seat.agent.clone();
3216            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217            self.state.seats.insert(seat.key.clone(), seat);
3218            let label = self.state.candidates[i].label;
3219            self.state.interrupted_implementations.remove(&label);
3220
3221            let (summary, duration, failed, verified_claim) = match out {
3222                AgentOutcome::Ok(o) => {
3223                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224                    let failed = (!o.usable()).then(|| {
3225                        if o.timed_out {
3226                            "agent timed out".to_owned()
3227                        } else {
3228                            format!(
3229                                "agent exited with {}",
3230                                crate::notices::exit_status(o.exit_code)
3231                            )
3232                        }
3233                    });
3234                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235                    (text, o.duration_ms, failed, verified_claim)
3236                }
3237                // Left un-resumed by `resume_undelivered` (a dirty tree
3238                // already rescues the work, or there was no session left to
3239                // resume into) — reported like the ordinary failure it is,
3240                // never as if `o.text` (the CLI's raw error JSON) were an
3241                // answer.
3242                AgentOutcome::Dropped(o) => {
3243                    let why = o
3244                        .dropped
3245                        .as_ref()
3246                        .map(|d| d.why.as_str())
3247                        .unwrap_or("the CLI ended the stream without delivering its answer");
3248                    (
3249                        String::new(),
3250                        o.duration_ms,
3251                        Some(format!("the CLI dropped the stream ({why})")),
3252                        None,
3253                    )
3254                }
3255                AgentOutcome::Quota(o) => {
3256                    self.state.quota.push(QuotaLoss::from_report(
3257                        seat_key,
3258                        "implement",
3259                        o.quota.as_ref(),
3260                    ));
3261                    (
3262                        String::new(),
3263                        o.duration_ms,
3264                        Some("rate limited (quota); produced no change".to_owned()),
3265                        None,
3266                    )
3267                }
3268                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269                    (String::new(), 0, Some(e), None)
3270                }
3271            };
3272
3273            // Save every completed turn before any fallible Git operation. A failed
3274            // rescue must not erase the reply or prevent collection of other seats.
3275            let c = &mut self.state.candidates[i];
3276            if !exhausted_the_fallback_chain {
3277                c.agent = agent;
3278            }
3279            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280            c.duration_ms = duration;
3281            c.failed = failed;
3282            c.verified_noop = verified_claim;
3283            self.state
3284                .candidate_preservation
3285                .insert(label, "awaiting persistence".to_owned());
3286        }
3287        self.state.save()?;
3288        self.persist_candidates().await?;
3289
3290        if resource_blocked {
3291            return self.stop_for_resource();
3292        }
3293        if selected_failed {
3294            self.state.status = RunStatus::Blocked;
3295            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296            self.state.save()?;
3297            bail!(
3298                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299            );
3300        }
3301        self.after_implement()
3302    }
3303
3304    /// Retry only persistence after a completed implementation turn.
3305    async fn persist_candidates(&mut self) -> Result<()> {
3306        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307        for label in labels {
3308            let i = self
3309                .state
3310                .candidates
3311                .iter()
3312                .position(|c| c.label == label)
3313                .context("pending candidate is missing")?;
3314            let c = self.state.candidates[i].clone();
3315            let interrupted = self.state.interrupted_implementations.contains(&label);
3316            let result = async {
3317                let wait = crate::repo_lock::WaitPolicy::new(
3318                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319                    || false,
3320                );
3321                let _coordination =
3322                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323                // Best effort evidence requires no index writes. A successful snapshot
3324                // never authorizes removal of a tree whose rescue failed.
3325                if let Err(e) = git::snapshot_uncommitted(
3326                    &c.worktree,
3327                    &self
3328                        .state
3329                        .dir()
3330                        .join("artifacts")
3331                        .join(format!("candidate-{label}-recovery")),
3332                )
3333                .await
3334                {
3335                    self.state.event(
3336                        "preserve",
3337                        format!("candidate {label} snapshot incomplete: {e:#}"),
3338                    );
3339                }
3340                let rescued = git::rescue_commit(
3341                    &c.worktree,
3342                    &format!("magi: candidate {label} (uncommitted work)"),
3343                )
3344                .await?;
3345                self.state.note_withheld("implement", &rescued.withheld);
3346                let base = &self.state.base_commit;
3347                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352                if c.empty && commits > 0 && !patch.trim().is_empty()
3353                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355                        || self.state.tally.is_some())
3356                {
3357                    // Keep the old classification and pending marker so every
3358                    // retry stops too. Retain decisions as recovery evidence;
3359                    // only a supported recovery operation may replace them.
3360                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361                }
3362                let c = &mut self.state.candidates[i];
3363                c.commits = commits;
3364                c.stat = stat;
3365                c.files = files;
3366                let no_changes = commits == 0 || patch.trim().is_empty();
3367                c.empty = no_changes && !interrupted;
3368                if interrupted && no_changes {
3369                    c.failed = None;
3370                    c.verified_noop = None;
3371                }
3372                if !c.empty {
3373                    c.failed = None;
3374                    c.verified_noop = None;
3375                }
3376                self.state.event(
3377                    "preserve",
3378                    format!("candidate {label}: {files} files, {commits} commits"),
3379                );
3380                Ok::<_, anyhow::Error>(())
3381            }
3382            .await;
3383            match result {
3384                Ok(()) => {
3385                    self.state.candidate_preservation.remove(&label);
3386                    if !interrupted || self.state.candidates[i].files > 0 {
3387                        self.state.interrupted_implementations.remove(&label);
3388                    }
3389                }
3390                Err(e) => {
3391                    let reason = format!("candidate {label} preservation failed: {e:#}");
3392                    self.state.event("preserve", &reason);
3393                    self.state.candidate_preservation.insert(label, reason);
3394                }
3395            }
3396            if self.state.candidate_preservation.is_empty()
3397                && self.state.status == RunStatus::Blocked
3398                && self.state.candidates.iter().all(|c| c.failed.is_none())
3399            {
3400                // Persist the recovered phase with the cleared pending marker;
3401                // a crash between saves must not turn this into a selected-seat
3402                // failure hold on the next resume.
3403                self.state.status = RunStatus::Implementing;
3404                self.state.event(
3405                    "preserve",
3406                    "candidate persistence recovered; implementation can continue",
3407                );
3408            }
3409            self.state.save()?;
3410        }
3411        if !self.state.candidate_preservation.is_empty() {
3412            self.state.status = RunStatus::Blocked;
3413            self.state.save()?;
3414            bail!(
3415                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416            );
3417        }
3418        Ok(())
3419    }
3420
3421    /// Ask again, once, for work a CLI did and then failed to hand over.
3422    ///
3423    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3424    /// status with an empty response and a usage report showing output tokens,
3425    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3426    /// seven minutes and 14,267 output tokens that arrived as an empty
3427    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3428    ///
3429    /// Two conditions, and both matter:
3430    ///
3431    /// - **Only when the tree is untouched.** Often the agent has already
3432    ///   written its files and only the closing message was lost; the rescue
3433    ///   commit below picks that up and there is nothing to ask for. Re-asking
3434    ///   then would pay for a second implementation of work already on disk.
3435    /// - **Once.** A CLI that drops one stream can drop the next, and this
3436    ///   node is the most expensive in the graph.
3437    ///
3438    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3439    /// dropped reply still carried its `conversation_id`, so the seat is asked
3440    /// to finish what it was doing rather than sent the whole task again. It
3441    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3442    /// node's - for the same reason a re-ranked judge does: restating finished
3443    /// work is not the work.
3444    ///
3445    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3446    /// same way until it resets, while an abandoned conversation is still
3447    /// there to be picked up.
3448    async fn resume_undelivered(
3449        &mut self,
3450        results: &mut [(usize, SeatState, AgentOutcome)],
3451        sent: &[SeatJob],
3452        prompts: &Prompts,
3453        run_id: &str,
3454    ) {
3455        for (wi, seat, out) in results.iter_mut() {
3456            let Some(dropped) = (match &*out {
3457                AgentOutcome::Dropped(o) => o.dropped.clone(),
3458                _ => None,
3459            }) else {
3460                continue;
3461            };
3462            let Some(job) = sent.get(*wi) else { continue };
3463            // Already on disk? Then only the closing message was lost.
3464            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465                self.state.event(
3466                    "implement",
3467                    format!(
3468                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469                         work is in the tree",
3470                        seat.key, dropped.output_tokens, dropped.why
3471                    ),
3472                );
3473                continue;
3474            }
3475            // The re-ask only makes sense as a resume: `resume_after_drop`
3476            // says nothing about the task, trusting the seat to still hold it.
3477            // Without a session to resume — sessions disabled, or this CLI's
3478            // drop shape happened not to carry a session id — that prompt
3479            // would open a brand-new conversation with no context at all,
3480            // which is worse than leaving this as the ordinary failure it
3481            // already is.
3482            if !has_context(&job.spec, seat, job.sessions) {
3483                self.state.event(
3484                    "implement",
3485                    format!(
3486                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487                         is no session left to resume",
3488                        seat.key, dropped.output_tokens, dropped.why
3489                    ),
3490                );
3491                continue;
3492            }
3493            self.state.event(
3494                "implement",
3495                format!(
3496                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497                     conversation",
3498                    seat.key, dropped.output_tokens, dropped.why
3499                ),
3500            );
3501            let mut retry = job.clone();
3502            retry.seat = seat.clone();
3503            retry.prompt = prompt::resume_after_drop(&dropped.why);
3504            retry.timeout = retry_budget(job.timeout, true);
3505            retry.stem = format!("{}-resume", job.stem);
3506            let cache = self.state.config.cache_dir();
3507            let ctx = WaveCtx {
3508                carry_seats: false,
3509                pauses: Some((&self.pause, &self.interrupt)),
3510                run: run_id,
3511                node: "implement",
3512                prompts,
3513                cache: cache.as_deref(),
3514                round: None,
3515            };
3516            let (resumed_seat, resumed) =
3517                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518            *seat = resumed_seat;
3519            *out = resumed;
3520        }
3521    }
3522
3523    /// Fall an implement seat through to the next untried agent in the
3524    /// implementer roster when it lost to quota — or, since the handover was
3525    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3526    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3527    /// quota path itself being unchanged — instead of leaving the
3528    /// seat's loss final the moment one agent's account runs dry.
3529    ///
3530    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3531    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3532    /// truncates to the single slot rotation picked, so a solo task whose one
3533    /// implementer hits quota mid-run used to have nothing else to try. This
3534    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3535    /// unrotated roster — which is the only place the *other* candidates in
3536    /// the machine's roster still exist once `implementers` has been cut down
3537    /// to size.
3538    ///
3539    /// Walks forward from just past the seat's own original position in the
3540    /// roster, never wrapping back to the front: a later candidate slot (say
3541    /// `beta`, the roster's second entry) must fall through to the *next*
3542    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3543    /// almost certainly a different candidate's own agent already — and once
3544    /// the roster's tail is exhausted there is nothing left to fall through
3545    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3546    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3547    /// let this retry that id forever. The loop keeps falling through until
3548    /// an attempt lands something other than `Quota` or the roster's tail
3549    /// runs out of untried ids, at which point the seat is left exactly as
3550    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3551    /// `QuotaLoss` recorded, the candidate failed/empty.
3552    ///
3553    /// `sent` is taken mutably and updated with the fallback agent's spec:
3554    /// `resume_unconfirmed_commands`, which runs after this and also reads
3555    /// `sent`, must see whichever agent actually ended up answering the seat
3556    /// — reading the stale, original spec there would check session
3557    /// eligibility against the wrong CLI and could hand a fallback agent's
3558    /// session id to the agent that just lost the seat to quota.
3559    ///
3560    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3561    /// — `self.seat` only reuses state when the agent id is unchanged, so
3562    /// handing it a different id already gets this for free. Reusing the old
3563    /// seat would resume a different CLI's session as if it were a
3564    /// continuation of this one.
3565    ///
3566    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3567    /// a quota loss cuts an agent off mid-turn, so anything already in the
3568    /// tree is unfinished work, not a completed candidate a re-ask would pay
3569    /// for twice. A dirty tree is rescued into a commit first (the same
3570    /// neutral-identity rescue `implement`'s own outcome loop gives every
3571    /// candidate) so the next agent starts clean.
3572    ///
3573    /// The new agent gets the implementer's full prompt and full
3574    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3575    /// it has no session and no context, and is implementing the task from
3576    /// nothing, unlike a resumed drop which is only restating work already
3577    /// done.
3578    ///
3579    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3580    /// `implement` event, never into `self.state.quota` — that is what
3581    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3582    /// go unspent, and a seat that ultimately recovered on its second or
3583    /// third agent is not the stalled panel that check exists to catch. Only
3584    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3585    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3586    /// outcome loop already has — this helper never pushes to it itself.
3587    async fn resume_seat_handovers(
3588        &mut self,
3589        results: &mut [(usize, SeatState, AgentOutcome)],
3590        sent: &mut [SeatJob],
3591        prompts: &Prompts,
3592        run_id: &str,
3593    ) -> Result<()> {
3594        let instruction = seeded_instruction(&self.state);
3595        let language = self.state.config.graph.language.clone();
3596        let github_language = self.state.config.graph.github_language.clone();
3597        let brief = self
3598            .state
3599            .advice
3600            .as_ref()
3601            .and_then(|a| a.synthesis.as_deref())
3602            .map(str::to_owned);
3603        let attachments = self.state.attachments.clone();
3604        for (wi, seat, out) in results.iter_mut() {
3605            // Who holds the other candidate seats of this wave right now
3606            // (earlier handovers already written back to `sent`).
3607            let others: BTreeSet<String> = sent
3608                .iter()
3609                .enumerate()
3610                .filter(|(j, _)| j != wi)
3611                .map(|(_, j)| j.spec.id.clone())
3612                .collect();
3613            let Some(job) = sent.get_mut(*wi) else {
3614                continue;
3615            };
3616            // Where the seat's own original agent sits in the roster — the
3617            // fallback walk starts just past here, never at the front, so a
3618            // later candidate slot's quota loss does not fall back onto an
3619            // earlier slot's own agent.
3620            let start = self
3621                .roles
3622                .implementer_roster
3623                .iter()
3624                .position(|s| s.id == job.spec.id)
3625                .unwrap_or(0);
3626            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627            let mut fallback_attempt = 0usize;
3628            let mut prev: Option<FailClass> = None;
3629            while let Some(cur) = FailClass::of(&*out) {
3630                if !should_hand_over(prev.as_ref(), &cur) {
3631                    break;
3632                }
3633                let Some(next) =
3634                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635                        .cloned()
3636                else {
3637                    break;
3638                };
3639                tried.insert(next.id.clone());
3640                fallback_attempt += 1;
3641
3642                let coordination = self.coordinate(&job.cwd).await?;
3643                if let Ok(r) = git::rescue_commit(
3644                    &job.cwd,
3645                    &format!(
3646                        "magi: candidate {} (uncommitted work before {} fallback)",
3647                        seat.key,
3648                        if cur == FailClass::Quota {
3649                            "quota"
3650                        } else {
3651                            "handover"
3652                        }
3653                    ),
3654                )
3655                .await
3656                {
3657                    self.state.note_withheld("implement", &r.withheld);
3658                }
3659                drop(coordination);
3660
3661                record_handover(
3662                    &mut self.state,
3663                    "implement",
3664                    &seat.key,
3665                    &seat.agent,
3666                    &next.id,
3667                    &cur,
3668                    &fail_reason(&*out),
3669                );
3670                prev = Some(cur.clone());
3671
3672                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674                // Kept in sync on `sent` itself, not just the local retry: a
3675                // later helper (`resume_unconfirmed_commands`) reads `sent`
3676                // after this one returns and must see whichever agent is now
3677                // occupying the seat, not the one that just quota'd out —
3678                // otherwise it would judge session/continuation eligibility
3679                // by the wrong CLI and could resend a fallback's session id
3680                // to the agent that lost it the seat in the first place.
3681                job.spec = next.clone();
3682                let mut retry = job.clone();
3683                retry.seat = new_seat;
3684                retry.prompt = prompt::implement_in(
3685                    &instruction,
3686                    &job.cwd.to_string_lossy(),
3687                    &language,
3688                    &github_language,
3689                    brief.as_deref(),
3690                    &attachments,
3691                );
3692                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693                let cache = self.state.config.cache_dir();
3694                let ctx = WaveCtx {
3695                    carry_seats: false,
3696                    pauses: Some((&self.pause, &self.interrupt)),
3697                    run: run_id,
3698                    node: "implement",
3699                    prompts,
3700                    cache: cache.as_deref(),
3701                    round: None,
3702                };
3703                let (fallback_seat, fallback_out) = run_one(
3704                    retry,
3705                    Arc::clone(&self.sem),
3706                    &ctx,
3707                    &mut self.state,
3708                    fallback_attempt,
3709                )
3710                .await;
3711                *seat = fallback_seat;
3712                *out = fallback_out;
3713            }
3714        }
3715        Ok(())
3716    }
3717
3718    /// Ask an implement seat's own CLI to confirm what it started, once, when
3719    /// its reply reported a command whose completion status it never
3720    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3721    /// that does and does not mean.
3722    ///
3723    /// The completion contract this task asks for, extended to `implement`
3724    /// with the same signal `continue_fix_report` reads for the fixer,
3725    /// rather than a keyword search over the reply or a hard requirement on
3726    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3727    /// a candidate's CLI turn ended cleanly while a test run it had started
3728    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3729    /// named in it at all is untouched by this: `commands` is empty, so
3730    /// there is nothing to be unconfirmed.
3731    ///
3732    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3733    /// this is not about recovering edits that might already be on disk, it
3734    /// is about a result the seat itself never vouched for, which resuming
3735    /// asks for regardless of what the tree already holds. Bounded to one
3736    /// attempt for the same reason `resume_undelivered` is — this is the
3737    /// most expensive node in the graph — and a seat that still cannot
3738    /// confirm on that attempt is left as whatever its (possibly still
3739    /// unconfirmed) reply says; this does not invent a new "failed" reason
3740    /// for a candidate that otherwise produced a real, committed change.
3741    async fn resume_unconfirmed_commands(
3742        &mut self,
3743        results: &mut [(usize, SeatState, AgentOutcome)],
3744        sent: &[SeatJob],
3745        prompts: &Prompts,
3746        run_id: &str,
3747    ) {
3748        for (wi, seat, out) in results.iter_mut() {
3749            let AgentOutcome::Ok(o) = &*out else {
3750                continue;
3751            };
3752            if !has_unconfirmed_command(&o.commands) {
3753                continue;
3754            }
3755            let Some(job) = sent.get(*wi) else { continue };
3756            if !has_context(&job.spec, seat, job.sessions) {
3757                self.state.event(
3758                    "implement",
3759                    format!(
3760                        "{}: the reply named a command whose own CLI never confirmed the exit \
3761                         status of, but there is no session left to resume",
3762                        seat.key
3763                    ),
3764                );
3765                continue;
3766            }
3767            self.state.event(
3768                "implement",
3769                format!(
3770                    "{}: the reply named a command whose own CLI never confirmed the exit \
3771                     status of; resuming the conversation",
3772                    seat.key
3773                ),
3774            );
3775            let mut retry = job.clone();
3776            retry.seat = seat.clone();
3777            retry.prompt = prompt::resume_incomplete(
3778                "a command in your last reply had no confirmed exit status",
3779            );
3780            retry.timeout = retry_budget(job.timeout, true);
3781            retry.stem = format!("{}-confirm", job.stem);
3782            let cache = self.state.config.cache_dir();
3783            let ctx = WaveCtx {
3784                carry_seats: false,
3785                pauses: Some((&self.pause, &self.interrupt)),
3786                run: run_id,
3787                node: "implement",
3788                prompts,
3789                cache: cache.as_deref(),
3790                round: None,
3791            };
3792            let (resumed_seat, resumed) =
3793                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794            *seat = resumed_seat;
3795            *out = resumed;
3796        }
3797    }
3798
3799    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3800    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3801    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3802    /// that motivated this.
3803    ///
3804    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3805    /// own nudge loop already covers for judge/review/vote seats, and not a
3806    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3807    /// implement seats: here the CLI turn genuinely finished while the node's
3808    /// own work — the fixer's account of what it did — had not. Gated purely
3809    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3810    /// reply, never on any wording in it, so a fixer whose valid, first-try
3811    /// `FixReport` happens to mention having waited on a background test is
3812    /// never resumed — the `Ok(report)` branch at the call site returns
3813    /// before this is ever invoked.
3814    ///
3815    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3816    /// attempt ([`retry_budget`]), nothing attempted once the session is
3817    /// gone, and a quota hit ends the loop immediately rather than retrying a
3818    /// rate limit that fails the same way again.
3819    async fn continue_fix_report(
3820        &mut self,
3821        mut seat: SeatState,
3822        parse_err: String,
3823        job: &SeatJob,
3824        prompts: &Prompts,
3825        run_id: &str,
3826        round: usize,
3827    ) -> (
3828        SeatState,
3829        Option<FixReport>,
3830        Option<String>,
3831        ContinuationRecord,
3832    ) {
3833        let mut last_err = parse_err;
3834        let mut cumulative_wait_ms = 0u64;
3835        let mut attempts = 0usize;
3836        loop {
3837            if !has_context(&job.spec, &seat, job.sessions) {
3838                self.state.event(
3839                    "fix",
3840                    format!(
3841                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842                         session left to resume into"
3843                    ),
3844                );
3845                let outcome = if attempts == 0 {
3846                    ContinuationOutcome::NoSession
3847                } else {
3848                    ContinuationOutcome::Exhausted
3849                };
3850                return (
3851                    seat,
3852                    None,
3853                    Some(format!("unparsable fix report: {last_err}")),
3854                    ContinuationRecord {
3855                        attempts,
3856                        cumulative_wait_ms,
3857                        outcome,
3858                    },
3859                );
3860            }
3861            if attempts >= MAX_FIX_CONTINUATIONS {
3862                self.state.event(
3863                    "fix",
3864                    format!(
3865                        "round {round}: fixer's reply still had no adoption report after \
3866                         {attempts} continuation(s) ({last_err}); giving up"
3867                    ),
3868                );
3869                return (
3870                    seat,
3871                    None,
3872                    Some(format!(
3873                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3874                    )),
3875                    ContinuationRecord {
3876                        attempts,
3877                        cumulative_wait_ms,
3878                        outcome: ContinuationOutcome::Exhausted,
3879                    },
3880                );
3881            }
3882            attempts += 1;
3883            self.state.event(
3884                "fix",
3885                format!(
3886                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888                ),
3889            );
3890            let mut retry = job.clone();
3891            retry.seat = seat.clone();
3892            retry.prompt = prompt::resume_incomplete(&last_err);
3893            retry.timeout = retry_budget(job.timeout, true);
3894            retry.stem = format!("{}-continue{attempts}", job.stem);
3895            let cache = self.state.config.cache_dir();
3896            let ctx = WaveCtx {
3897                carry_seats: false,
3898                pauses: Some((&self.pause, &self.interrupt)),
3899                run: run_id,
3900                node: "fix",
3901                prompts,
3902                cache: cache.as_deref(),
3903                round: Some(round),
3904            };
3905            let (resumed_seat, resumed_out) = run_one(
3906                retry,
3907                Arc::clone(&self.sem),
3908                &ctx,
3909                &mut self.state,
3910                attempts,
3911            )
3912            .await;
3913            seat = resumed_seat;
3914            match resumed_out {
3915                AgentOutcome::Ok(o) => {
3916                    cumulative_wait_ms += o.duration_ms;
3917                    match verdict::extract_json::<FixReport>(&o.text) {
3918                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919                            self.state.event(
3920                                "fix",
3921                                format!(
3922                                    "round {round}: fixer's adoption report recovered after \
3923                                     {attempts} continuation(s)"
3924                                ),
3925                            );
3926                            return (
3927                                seat,
3928                                Some(report),
3929                                None,
3930                                ContinuationRecord {
3931                                    attempts,
3932                                    cumulative_wait_ms,
3933                                    outcome: ContinuationOutcome::Resumed,
3934                                },
3935                            );
3936                        }
3937                        // The report parsed, but this same reply's own
3938                        // CommandEvidence — the identical record `state.jobs`
3939                        // renders — names a command whose CLI never
3940                        // confirmed an exit status. Read together, that is
3941                        // not a resolved answer: keep nudging rather than
3942                        // accept a report standing next to a command the
3943                        // seat's own CLI cannot vouch for.
3944                        Ok(_) => {
3945                            last_err = "the reply parsed, but it reported a command whose own CLI \
3946                                 never confirmed an exit status"
3947                                .to_owned();
3948                        }
3949                        Err(e) => last_err = e.to_string(),
3950                    }
3951                }
3952                AgentOutcome::Quota(o) => {
3953                    cumulative_wait_ms += o.duration_ms;
3954                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956                    self.state.quota.push(loss);
3957                    self.state.event(
3958                        "fix",
3959                        format!(
3960                            "round {round}: continuation rate limited (quota); not retrying now"
3961                        ),
3962                    );
3963                    return (
3964                        seat,
3965                        None,
3966                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3967                        ContinuationRecord {
3968                            attempts,
3969                            cumulative_wait_ms,
3970                            outcome: ContinuationOutcome::QuotaLost,
3971                        },
3972                    );
3973                }
3974                AgentOutcome::Dropped(o) => {
3975                    cumulative_wait_ms += o.duration_ms;
3976                    let why = o
3977                        .dropped
3978                        .as_ref()
3979                        .map(|d| d.why.as_str())
3980                        .unwrap_or("the CLI ended the stream without delivering its answer");
3981                    last_err = format!("the CLI dropped the stream ({why})");
3982                }
3983                AgentOutcome::Failed(e) => last_err = e,
3984                // The continuation was never started; the report stays lost
3985                // and the caller records that, without any handover.
3986                AgentOutcome::ResourceBlocked(e) => {
3987                    return (
3988                        seat,
3989                        None,
3990                        Some(format!("the continuation was not started: {e}")),
3991                        ContinuationRecord {
3992                            attempts: attempts.saturating_sub(1),
3993                            cumulative_wait_ms,
3994                            outcome: ContinuationOutcome::NoSession,
3995                        },
3996                    );
3997                }
3998            }
3999        }
4000    }
4001
4002    fn after_implement(&mut self) -> Result<()> {
4003        if !self.state.candidate_preservation.is_empty() {
4004            bail!("candidate preservation incomplete; judging is unsafe");
4005        }
4006        // Scan every candidate patch once the set is complete.
4007        if self.state.leaks.is_empty() {
4008            let cfg = self.state.config.blind.clone();
4009            let mut leaks = Vec::new();
4010            for c in &self.state.candidates {
4011                let Some(patch) =
4012                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013                else {
4014                    continue;
4015                };
4016                leaks.extend(blind::scan(
4017                    &format!("candidate {} patch", c.label),
4018                    &patch,
4019                    &cfg.vendor_tokens,
4020                ));
4021            }
4022            if !leaks.is_empty() {
4023                let summary = leaks
4024                    .iter()
4025                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026                    .collect::<Vec<_>>()
4027                    .join(", ");
4028                match cfg.on_leak {
4029                    LeakPolicy::Fail => {
4030                        self.state.status = RunStatus::Failed;
4031                        self.state
4032                            .event("blind", format!("vendor text in a patch: {summary}"));
4033                        self.state.leaks = leaks;
4034                        self.state.save()?;
4035                        self.settle_questions();
4036                        bail!(
4037                            "blind.on_leak = \"fail\" and vendor text reached a \
4038                             judged patch: {summary}"
4039                        );
4040                    }
4041                    LeakPolicy::Redact => self.state.event(
4042                        "blind",
4043                        format!("redacting vendor text for judging: {summary}"),
4044                    ),
4045                    LeakPolicy::Warn => self.state.event(
4046                        "blind",
4047                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048                    ),
4049                }
4050                self.state.leaks = leaks;
4051            }
4052        }
4053
4054        if self.state.viable().is_empty() {
4055            if self.state.all_candidates_verified_noop() {
4056                // Every candidate agreed, with evidence the adoption guard
4057                // accepted, that nothing belongs in this worktree. That is
4058                // not the same fact as a candidate that simply failed to
4059                // write anything, and settling it as an ordinary `Failed`
4060                // (see `SCHEMA`'s doc for schema 10) is what let two of
4061                // task 391f's attempts burn a retry each re-discovering the
4062                // same already-landed fix. Terminal either way, so `judge`
4063                // must never run over an empty candidate set — unlike the
4064                // `Failed` branch below this returns `Ok`, not an error:
4065                // nothing here failed.
4066                self.state.status = RunStatus::VerifiedNoop;
4067                self.state.save()?;
4068                self.settle_questions();
4069                return Ok(());
4070            }
4071            self.state.status = RunStatus::Failed;
4072            self.state.save()?;
4073            self.settle_questions();
4074            bail!("no candidate produced a change; nothing to judge");
4075        }
4076        self.state.status = RunStatus::Judging;
4077        self.state.save()?;
4078        Ok(())
4079    }
4080
4081    // --------------------------------------------------------------- judge
4082
4083    async fn judge(&mut self) -> Result<()> {
4084        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4085        // agent files with `magi task add` name the run that paid for it. The
4086        // prompt overlay is cloned alongside it because the waves borrow it
4087        // while `self` is mutably borrowed by the node's own bookkeeping.
4088        let run_id = self.state.id.clone();
4089        let prompts = self.state.config.prompts.clone();
4090        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091            return Ok(());
4092        }
4093        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094        if viable.len() == 1 {
4095            // Recorded so this is a one-time event: `judgements` stays empty
4096            // either way, which without this flag is indistinguishable from
4097            // "not yet judged" on the next reentry — and status is left
4098            // untouched, so a later node's conclusion (e.g. `Blocked` after
4099            // the review budget ran out) survives a resume instead of being
4100            // clobbered back to `Judging` by this node running again.
4101            self.state.judge_skipped = true;
4102            self.state.event(
4103                "judge",
4104                format!(
4105                    "only candidate {} produced a change; judging skipped",
4106                    viable[0].label
4107                ),
4108            );
4109            self.state.save()?;
4110            return Ok(());
4111        }
4112        self.state.status = RunStatus::Judging;
4113
4114        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115        let language = self.state.config.graph.language.clone();
4116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117        let sessions = self.state.config.graph.sessions;
4118        let artifacts = agent::artifacts_dir(&self.state.dir());
4119        let root = self.state.worktree_root();
4120        let base_short = short(&self.state.base_commit);
4121
4122        let mut jobs = Vec::new();
4123        let mut orders = Vec::new();
4124        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128            let seat_key = format!("judge-{}", j + 1);
4129            let seat = self.seat(&seat_key, &spec.id);
4130            jobs.push(SeatJob {
4131                prompt: prompt::judge(
4132                    &self.state.instruction,
4133                    &views,
4134                    self.roles.judges.len(),
4135                    &base_short,
4136                    &language,
4137                ),
4138                spec,
4139                seat,
4140                cwd: root.join(format!("judge-{}", j + 1)),
4141                timeout,
4142                allow_write: false,
4143                sessions,
4144                artifacts: artifacts.clone(),
4145                stem: format!("judge-{}", j + 1),
4146                handover: None,
4147            });
4148        }
4149
4150        self.state.event(
4151            "judge",
4152            format!(
4153                "{} judges ranking {} candidates blind",
4154                jobs.len(),
4155                viable.len()
4156            ),
4157        );
4158        let labels_for_check = labels.clone();
4159        let mut quota_losses = Vec::new();
4160        let cache = self.state.config.cache_dir();
4161        let ctx = WaveCtx {
4162            carry_seats: false,
4163            pauses: Some((&self.pause, &self.interrupt)),
4164            run: &run_id,
4165            node: "judge",
4166            prompts: &prompts,
4167            cache: cache.as_deref(),
4168            round: None,
4169        };
4170        let results = ask_json_wave::<Ranking>(
4171            jobs,
4172            Arc::clone(&self.sem),
4173            self.state.config.graph.retries,
4174            &self.roles.judge_roster,
4175            &ctx,
4176            &mut quota_losses,
4177            &mut self.state,
4178            &move |r: &Ranking| r.validate(&labels_for_check),
4179        )
4180        .await;
4181        if self.state.resource_block.is_some() {
4182            return self.stop_for_resource();
4183        }
4184        self.state.quota.extend(quota_losses);
4185
4186        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187            let agent_id = seat.agent.clone();
4188            self.state.seats.insert(seat.key.clone(), seat);
4189            let mut record = Judgement {
4190                judge: j + 1,
4191                seat: format!("judge-{}", j + 1),
4192                agent: agent_id,
4193                ranking: Vec::new(),
4194                reasons: BTreeMap::new(),
4195                confidence: None,
4196                strengths: BTreeMap::new(),
4197                order: orders[j].clone(),
4198                failed: None,
4199                duration_ms: 0,
4200            };
4201            match res {
4202                Ok((ranking, out)) => {
4203                    record.ranking = ranking.normalized();
4204                    record.reasons = ranking.reasons;
4205                    record.strengths = ranking.strengths;
4206                    record.confidence = ranking.confidence;
4207                    record.duration_ms = out.duration_ms;
4208                    self.state.event(
4209                        "judge",
4210                        format!(
4211                            "judge {} ranked {}",
4212                            j + 1,
4213                            record.ranking.iter().collect::<String>()
4214                        ),
4215                    );
4216                }
4217                Err(e) => {
4218                    record.failed = Some(e.to_string());
4219                    self.state
4220                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221                }
4222            }
4223            self.state.judgements.push(record);
4224            self.state.save()?;
4225        }
4226        Ok(())
4227    }
4228
4229    // ---------------------------------------------------------- deliberate
4230
4231    async fn deliberate(&mut self) -> Result<()> {
4232        if self.hold_base_sync().await? {
4233            return Ok(());
4234        }
4235        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4236        // agent files with `magi task add` name the run that paid for it. The
4237        // prompt overlay is cloned alongside it because the waves borrow it
4238        // while `self` is mutably borrowed by the node's own bookkeeping.
4239        let run_id = self.state.id.clone();
4240        let prompts = self.state.config.prompts.clone();
4241        let rounds = self.state.config.graph.deliberate_rounds;
4242        // Resume unfinished rounds after an outage, but never reopen a tally.
4243        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244            return Ok(());
4245        }
4246        let tops: Vec<char> = self
4247            .state
4248            .judgements
4249            .iter()
4250            .filter_map(|j| j.ranking.first().copied())
4251            .collect();
4252        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254                self.state.event(
4255                    "deliberate",
4256                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4257                );
4258            }
4259            self.state.status = RunStatus::Voting;
4260            self.state.save()?;
4261            return Ok(());
4262        }
4263
4264        self.state.status = RunStatus::Deliberating;
4265        self.state.event(
4266            "deliberate",
4267            format!(
4268                "split: first choices were {} — opening {rounds} round(s)",
4269                tops.iter().collect::<String>()
4270            ),
4271        );
4272
4273        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274        let language = self.state.config.graph.language.clone();
4275        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276        let sessions = self.state.config.graph.sessions;
4277        let artifacts = agent::artifacts_dir(&self.state.dir());
4278        let root = self.state.worktree_root();
4279        let base_short = short(&self.state.base_commit);
4280
4281        // Judges argue in sequence so that a turn can answer the one before it;
4282        // that is the difference between deliberation and three parallel
4283        // monologues.
4284        for round in self.state.deliberation.len() + 1..=rounds {
4285            let mut turns: Vec<DeliberationTurn> = Vec::new();
4286            let mut round_losses = Vec::new();
4287            let mut asked = 0;
4288            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289                if self.state.judgements[j].failed.is_some() {
4290                    continue;
4291                }
4292                let seat_key = format!("judge-{}", j + 1);
4293                let spec = self.occupant(&seat_key, spec);
4294                let mut seat = self.seat(&seat_key, &spec.id);
4295                let transcript = self.transcript(&turns, j);
4296                let build = |context: Option<&str>| {
4297                    prompt::deliberate(
4298                        &self.state.instruction,
4299                        context,
4300                        &transcript,
4301                        round,
4302                        rounds,
4303                        &language,
4304                    )
4305                };
4306                let block = self.candidate_block(&viable, &base_short);
4307                let full = build(Some(&block));
4308                let text = if has_context(&spec, &seat, sessions) {
4309                    build(None)
4310                } else {
4311                    full.clone()
4312                };
4313                let job = SeatJob {
4314                    spec,
4315                    seat: seat.clone(),
4316                    prompt: text,
4317                    cwd: root.join(format!("judge-{}", j + 1)),
4318                    timeout,
4319                    allow_write: false,
4320                    sessions,
4321                    artifacts: artifacts.clone(),
4322                    stem: format!("delib-{round}-judge-{}", j + 1),
4323                    handover: Some(full),
4324                };
4325                let cache = self.state.config.cache_dir();
4326                let ctx = WaveCtx {
4327                    carry_seats: false,
4328                    pauses: Some((&self.pause, &self.interrupt)),
4329                    run: &run_id,
4330                    node: "deliberate",
4331                    prompts: &prompts,
4332                    cache: cache.as_deref(),
4333                    round: None,
4334                };
4335                // A turn is never nudged (`retries` 0): a failed seat is
4336                // handed to the next roster agent, which gets the full
4337                // context. An empty answer is a turn, not a failure.
4338                let mut losses = Vec::new();
4339                asked += 1;
4340                let mut results = ask_wave_with::<String>(
4341                    vec![job],
4342                    Arc::clone(&self.sem),
4343                    0,
4344                    &self.roles.judge_roster,
4345                    &ctx,
4346                    &mut losses,
4347                    &mut self.state,
4348                    &|text: &str| {
4349                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350                    },
4351                )
4352                .await;
4353                if self.state.resource_block.is_some() {
4354                    return self.stop_for_resource();
4355                }
4356                round_losses.extend(losses.iter().cloned());
4357                self.state.quota.extend(losses);
4358                let (updated, res, _) = results.pop().expect("one job in, one result out");
4359                seat = updated;
4360                let agent_id = seat.agent.clone();
4361                self.state.seats.insert(seat.key.clone(), seat);
4362                let body = match res {
4363                    Ok((body, _)) => body,
4364                    // Skip the seat; a CLI's raw error JSON is never read as
4365                    // this judge's position.
4366                    Err(e) => {
4367                        self.state
4368                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369                        continue;
4370                    }
4371                };
4372                let tentative = verdict::extract_json::<Position>(&body)
4373                    .ok()
4374                    .and_then(|p| p.tentative)
4375                    .and_then(|s| s.trim().chars().next())
4376                    .map(|c| c.to_ascii_uppercase());
4377                self.state.event(
4378                    "deliberate",
4379                    format!(
4380                        "round {round}: judge {} now favours {}",
4381                        j + 1,
4382                        tentative.map_or("—".to_owned(), |c| c.to_string())
4383                    ),
4384                );
4385                turns.push(DeliberationTurn {
4386                    judge: j + 1,
4387                    agent: agent_id,
4388                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4389                    tentative,
4390                });
4391            }
4392            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394                return self.stop_for_resource();
4395            }
4396            self.state
4397                .deliberation
4398                .push(DeliberationRound { round, turns });
4399            self.state.save()?;
4400        }
4401
4402        self.state.status = RunStatus::Voting;
4403        self.state.save()?;
4404        Ok(())
4405    }
4406
4407    // ---------------------------------------------------------------- vote
4408
4409    async fn vote(&mut self) -> Result<()> {
4410        if self.hold_base_sync().await? {
4411            return Ok(());
4412        }
4413        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4414        // agent files with `magi task add` name the run that paid for it. The
4415        // prompt overlay is cloned alongside it because the waves borrow it
4416        // while `self` is mutably borrowed by the node's own bookkeeping.
4417        let run_id = self.state.id.clone();
4418        let prompts = self.state.config.prompts.clone();
4419        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420            return Ok(());
4421        }
4422        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423        if viable.len() == 1 {
4424            return Ok(());
4425        }
4426        self.state.status = RunStatus::Voting;
4427
4428        let language = self.state.config.graph.language.clone();
4429        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430        let sessions = self.state.config.graph.sessions;
4431        let artifacts = agent::artifacts_dir(&self.state.dir());
4432        let root = self.state.worktree_root();
4433        let base_short = short(&self.state.base_commit);
4434        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436        let mut jobs = Vec::new();
4437        let mut seats_at = Vec::new();
4438        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439            if self
4440                .state
4441                .judgements
4442                .get(j)
4443                .is_some_and(|r| r.failed.is_some())
4444            {
4445                continue;
4446            }
4447            let seat_key = format!("judge-{}", j + 1);
4448            let spec = self.occupant(&seat_key, spec);
4449            let seat = self.seat(&seat_key, &spec.id);
4450            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451            let text = if has_context(&spec, &seat, sessions) {
4452                prompt::final_vote(&viable, &language)
4453            } else {
4454                full.clone()
4455            };
4456            jobs.push(SeatJob {
4457                spec,
4458                seat,
4459                prompt: text,
4460                cwd: root.join(format!("judge-{}", j + 1)),
4461                timeout,
4462                allow_write: false,
4463                sessions,
4464                artifacts: artifacts.clone(),
4465                stem: format!("vote-judge-{}", j + 1),
4466                handover: Some(full),
4467            });
4468            seats_at.push(j);
4469        }
4470
4471        self.state.event(
4472            "vote",
4473            format!(
4474                "collecting {} final votes one by one, privately",
4475                jobs.len()
4476            ),
4477        );
4478        let allowed = viable.clone();
4479        let mut quota_losses = Vec::new();
4480        let cache = self.state.config.cache_dir();
4481        let ctx = WaveCtx {
4482            carry_seats: false,
4483            pauses: Some((&self.pause, &self.interrupt)),
4484            run: &run_id,
4485            node: "vote",
4486            prompts: &prompts,
4487            cache: cache.as_deref(),
4488            round: None,
4489        };
4490        let results = ask_json_wave::<FinalVote>(
4491            jobs,
4492            Arc::clone(&self.sem),
4493            self.state.config.graph.retries,
4494            &self.roles.judge_roster,
4495            &ctx,
4496            &mut quota_losses,
4497            &mut self.state,
4498            &move |v: &FinalVote| match v.label() {
4499                Some(c) if allowed.contains(&c) => Ok(()),
4500                other => bail!("vote {other:?} is not one of {allowed:?}"),
4501            },
4502        )
4503        .await;
4504        if self.state.resource_block.is_some() {
4505            return self.stop_for_resource();
4506        }
4507        self.state.quota.extend(quota_losses);
4508
4509        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510            let agent_id = seat.agent.clone();
4511            self.state.seats.insert(seat.key.clone(), seat);
4512            let initial = self
4513                .state
4514                .judgements
4515                .get(j)
4516                .and_then(|r| r.ranking.first().copied());
4517            let mut record = VoteRecord {
4518                judge: j + 1,
4519                agent: agent_id,
4520                vote: None,
4521                reason: String::new(),
4522                changed: false,
4523            };
4524            match res {
4525                Ok((v, _)) => {
4526                    record.vote = v.label();
4527                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529                    self.state.event(
4530                        "vote",
4531                        format!(
4532                            "judge {} voted {}{}",
4533                            j + 1,
4534                            record.vote.unwrap_or('?'),
4535                            if record.changed { " (changed)" } else { "" }
4536                        ),
4537                    );
4538                }
4539                Err(e) => {
4540                    self.state
4541                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542                }
4543            }
4544            self.state.votes.push(record);
4545            self.state.save()?;
4546        }
4547        Ok(())
4548    }
4549
4550    // --------------------------------------------------------------- tally
4551
4552    fn tally(&mut self) -> Result<()> {
4553        if self.state.tally.is_some() {
4554            return Ok(());
4555        }
4556        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557        let tops: Vec<char> = self
4558            .state
4559            .judgements
4560            .iter()
4561            .filter_map(|j| j.ranking.first().copied())
4562            .collect();
4563        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565        // A judge whose private vote failed still counted once, in the initial
4566        // ranking; using it beats discarding a whole seat.
4567        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568        let mut cast: Vec<char> = Vec::new();
4569        for (i, j) in self.state.judgements.iter().enumerate() {
4570            let vote = self
4571                .state
4572                .votes
4573                .iter()
4574                .find(|v| v.judge == i + 1)
4575                .and_then(|v| v.vote)
4576                .or_else(|| j.ranking.first().copied());
4577            if let Some(v) = vote {
4578                *first_choice.entry(v).or_insert(0) += 1;
4579                cast.push(v);
4580            }
4581        }
4582
4583        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584        for j in &self.state.judgements {
4585            let n = j.ranking.len();
4586            for (pos, label) in j.ranking.iter().enumerate() {
4587                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588            }
4589        }
4590
4591        let best = first_choice.values().copied().max().unwrap_or(0);
4592        let mut leaders: Vec<char> = first_choice
4593            .iter()
4594            .filter(|(_, v)| **v == best)
4595            .map(|(k, _)| *k)
4596            .collect();
4597        let mut tie_break = None;
4598        if leaders.len() > 1 {
4599            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600            let borda_leaders: Vec<char> = leaders
4601                .iter()
4602                .copied()
4603                .filter(|l| borda[l] == top_borda)
4604                .collect();
4605            tie_break = Some(if borda_leaders.len() == 1 {
4606                format!(
4607                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608                    leaders.len()
4609                )
4610            } else {
4611                format!(
4612                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4613                    leaders.len()
4614                )
4615            });
4616            leaders = borda_leaders;
4617            leaders.sort_unstable();
4618        }
4619        let winner = *leaders
4620            .first()
4621            .or(viable.first())
4622            .context("no candidate to declare a winner from")?;
4623
4624        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626        let deliberated = !self.state.deliberation.is_empty();
4627
4628        // Whose verdict is this? A rate-limited seat is absent even if it
4629        // ranked before the limit hit, so presence is measured against the
4630        // recorded losses, not just "did a ranking ever appear".
4631        let quota_seats: std::collections::BTreeSet<&str> =
4632            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633        let mut present = 0usize;
4634        for (i, j) in self.state.judgements.iter().enumerate() {
4635            if quota_seats.contains(j.seat.as_str()) {
4636                continue;
4637            }
4638            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639            let voted = self
4640                .state
4641                .votes
4642                .iter()
4643                .any(|v| v.judge == i + 1 && v.vote.is_some());
4644            if ranked || voted {
4645                present += 1;
4646            }
4647        }
4648        // Strict majority of the configured panel. A bare majority is real
4649        // signal we can act on, while a minority verdict must never stand in
4650        // for a healthy one. A one-candidate run needs no panel at all, and
4651        // `judges` stays `0` rather than the roster size a panel that never
4652        // sat would otherwise be credited with.
4653        let needs_quorum = viable.len() > 1;
4654        let judges_total = if needs_quorum {
4655            self.roles.judges.len()
4656        } else {
4657            0
4658        };
4659        let quorum = if needs_quorum {
4660            judges_total / 2 + 1
4661        } else {
4662            0
4663        };
4664        let met_quorum = !needs_quorum || present >= quorum;
4665        let uncontested = (!needs_quorum).then(|| {
4666            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667        });
4668
4669        self.state.event(
4670            "tally",
4671            match &uncontested {
4672                Some(reason) => format!("winner {winner} — {reason}"),
4673                None => format!(
4674                    "winner {winner} — votes {} | initial {} | {} changed | \
4675                     {present}/{judges_total} judges{}",
4676                    first_choice
4677                        .iter()
4678                        .map(|(k, v)| format!("{k}:{v}"))
4679                        .collect::<Vec<_>>()
4680                        .join(" "),
4681                    if unanimous_initial {
4682                        "unanimous"
4683                    } else {
4684                        "split"
4685                    },
4686                    changed_votes,
4687                    if met_quorum {
4688                        String::new()
4689                    } else {
4690                        format!(" — below quorum ({quorum} required)")
4691                    },
4692                ),
4693            },
4694        );
4695        if !met_quorum {
4696            self.state.event(
4697                "stall",
4698                format!(
4699                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700                     the run stops here, resumable"
4701                ),
4702            );
4703        }
4704        self.state.tally = Some(Tally {
4705            first_choice,
4706            borda,
4707            winner,
4708            rankings: tops.len(),
4709            unanimous_initial,
4710            deliberated,
4711            changed_votes,
4712            unanimous_final,
4713            tie_break,
4714            judges: judges_total,
4715            present,
4716            quorum,
4717            met_quorum,
4718            uncontested,
4719        });
4720        self.state.status = if met_quorum {
4721            RunStatus::Reviewing
4722        } else {
4723            RunStatus::Stalled
4724        };
4725        self.state.save()?;
4726        Ok(())
4727    }
4728
4729    // ------------------------------------------------------------- recover
4730
4731    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4732    /// resumed toward completion once the transient cause clears.
4733    ///
4734    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4735    /// it toward the quorum, which is exactly the set of seats whose absence
4736    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4737    /// not depend on which node happened to hit the limit), or an ordinary
4738    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4739    /// seat is never disturbed.
4740    ///
4741    /// A seat that now answers with a usable ranking is "recovered": its
4742    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4743    /// `tally` counts it present again), and its vote re-collected. A seat that
4744    /// still fails keeps its loss and stays absent.
4745    ///
4746    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4747    /// to review/gate/merge), `false` when it is still below quorum (the run
4748    /// stays `Stalled`, still resumable for a later retry).
4749    #[allow(clippy::too_many_lines)]
4750    async fn recover_stall(&mut self) -> Result<bool> {
4751        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4752        // agent files with `magi task add` name the run that paid for it. The
4753        // prompt overlay is cloned alongside it because the waves borrow it
4754        // while `self` is mutably borrowed by the node's own bookkeeping.
4755        let run_id = self.state.id.clone();
4756        let prompts = self.state.config.prompts.clone();
4757        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4758        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4759        // is what keeps a non-quota collapse — or a quota loss recorded at the
4760        // deliberate node — from being a permanent dead-end on `--resume`.
4761        let quota_seats: BTreeSet<&str> =
4762            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763        let absent: Vec<String> = self
4764            .state
4765            .judgements
4766            .iter()
4767            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768            .map(|j| j.seat.clone())
4769            .collect();
4770        if absent.is_empty() {
4771            return Ok(false);
4772        }
4773        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774        if viable.len() <= 1 {
4775            return Ok(false);
4776        }
4777        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778        let language = self.state.config.graph.language.clone();
4779        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780        let sessions = self.state.config.graph.sessions;
4781        let artifacts = agent::artifacts_dir(&self.state.dir());
4782        let root = self.state.worktree_root();
4783        let base_short = short(&self.state.base_commit);
4784        let candidates: Vec<Candidate> = viable.clone();
4785
4786        // Map each absent seat key to its 0-based position in `roles.judges`.
4787        let mut positions: Vec<usize> = absent
4788            .iter()
4789            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790            .collect();
4791        if positions.is_empty() {
4792            return Ok(false);
4793        }
4794        positions.sort_unstable();
4795        positions.dedup();
4796
4797        // Re-rank the lost seats, one blind prompt each.
4798        let mut judge_jobs = Vec::new();
4799        for &j in &positions {
4800            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802            let seat_key = format!("judge-{}", j + 1);
4803            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804            let seat = self.seat(&seat_key, &spec.id);
4805            judge_jobs.push(SeatJob {
4806                spec,
4807                seat,
4808                prompt: prompt::judge(
4809                    &self.state.instruction,
4810                    &views,
4811                    self.roles.judges.len(),
4812                    &base_short,
4813                    &language,
4814                ),
4815                cwd: root.join(seat_key),
4816                timeout,
4817                allow_write: false,
4818                sessions,
4819                artifacts: artifacts.clone(),
4820                stem: format!("judge-{}-recover", j + 1),
4821                handover: None,
4822            });
4823        }
4824
4825        let labels_for_check = labels.clone();
4826        let mut judge_losses = Vec::new();
4827        let retries = self.state.config.graph.retries;
4828        let cache = self.state.config.cache_dir();
4829        let ctx = WaveCtx {
4830            carry_seats: false,
4831            pauses: Some((&self.pause, &self.interrupt)),
4832            run: &run_id,
4833            node: "judge",
4834            prompts: &prompts,
4835            cache: cache.as_deref(),
4836            round: None,
4837        };
4838        let results = ask_json_wave::<Ranking>(
4839            judge_jobs,
4840            Arc::clone(&self.sem),
4841            retries,
4842            &self.roles.judge_roster,
4843            &ctx,
4844            &mut judge_losses,
4845            &mut self.state,
4846            &move |r: &Ranking| r.validate(&labels_for_check),
4847        )
4848        .await;
4849        if self.state.resource_block.is_some() {
4850            self.stop_for_resource()?;
4851            return Ok(false);
4852        }
4853
4854        // Refresh the judgement of every seat that ranked again.
4855        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857            let agent_id = seat.agent.clone();
4858            self.state.seats.insert(seat.key.clone(), seat);
4859            let record = &mut self.state.judgements[j];
4860            match res {
4861                Ok((ranking, out)) => {
4862                    record.agent = agent_id;
4863                    record.ranking = ranking.normalized();
4864                    record.reasons = ranking.reasons;
4865                    record.strengths = ranking.strengths;
4866                    record.confidence = ranking.confidence;
4867                    record.failed = None;
4868                    record.duration_ms = out.duration_ms;
4869                    recovered.insert(j);
4870                    self.state.event(
4871                        "recover",
4872                        format!("judge {} ranked again after the limit", j + 1),
4873                    );
4874                }
4875                Err(e) => {
4876                    self.state
4877                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878                }
4879            }
4880        }
4881
4882        // Re-ask the votes of the seats that recovered a ranking.
4883        let mut vote_jobs = Vec::new();
4884        let mut vote_pos: Vec<usize> = Vec::new();
4885        for &j in &recovered {
4886            let seat_key = format!("judge-{}", j + 1);
4887            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888            let seat = self.seat(&seat_key, &spec.id);
4889            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890            let text = if has_context(&spec, &seat, sessions) {
4891                prompt::final_vote(&labels, &language)
4892            } else {
4893                full.clone()
4894            };
4895            vote_jobs.push(SeatJob {
4896                spec,
4897                seat,
4898                prompt: text,
4899                cwd: root.join(seat_key),
4900                timeout,
4901                allow_write: false,
4902                sessions,
4903                artifacts: artifacts.clone(),
4904                stem: format!("vote-judge-{}-recover", j + 1),
4905                handover: Some(full),
4906            });
4907            vote_pos.push(j);
4908        }
4909        let allowed = labels.clone();
4910        let mut vote_losses = Vec::new();
4911        let vote_retries = self.state.config.graph.retries;
4912        let vote_cache = self.state.config.cache_dir();
4913        let ctx = WaveCtx {
4914            carry_seats: false,
4915            pauses: Some((&self.pause, &self.interrupt)),
4916            run: &run_id,
4917            node: "vote",
4918            prompts: &prompts,
4919            cache: vote_cache.as_deref(),
4920            round: None,
4921        };
4922        let votes = ask_json_wave::<FinalVote>(
4923            vote_jobs,
4924            Arc::clone(&self.sem),
4925            vote_retries,
4926            &self.roles.judge_roster,
4927            &ctx,
4928            &mut vote_losses,
4929            &mut self.state,
4930            &move |v: &FinalVote| match v.label() {
4931                Some(c) if allowed.contains(&c) => Ok(()),
4932                other => bail!("vote {other:?} is not one of {allowed:?}"),
4933            },
4934        )
4935        .await;
4936        if self.state.resource_block.is_some() {
4937            self.stop_for_resource()?;
4938            return Ok(false);
4939        }
4940        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941            let agent_id = seat.agent.clone();
4942            self.state.seats.insert(seat.key.clone(), seat);
4943            match res {
4944                Ok((v, _)) => {
4945                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946                        rec.vote = v.label();
4947                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948                    } else {
4949                        self.state.votes.push(VoteRecord {
4950                            judge: j + 1,
4951                            agent: agent_id,
4952                            vote: v.label(),
4953                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954                            changed: false,
4955                        });
4956                    }
4957                    self.state.event(
4958                        "recover",
4959                        format!("judge {} voted again after the limit", j + 1),
4960                    );
4961                }
4962                Err(e) => {
4963                    self.state
4964                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965                }
4966            }
4967        }
4968
4969        // A seat that ranked again is present even if its re-vote failed —
4970        // `tally` falls back to the initial ranking's first choice — so clear
4971        // its quota loss. Seats that still fail keep theirs and stay absent.
4972        let recovered_keys: BTreeSet<String> = recovered
4973            .iter()
4974            .map(|&j| format!("judge-{}", j + 1))
4975            .collect();
4976        self.state
4977            .quota
4978            .retain(|q| !recovered_keys.contains(&q.seat));
4979        // A seat that hit the limit again is a fresh loss, not the old one:
4980        // replace the stale entry so the history stays one-per-seat and the
4981        // daemon can tell this attempt's loss from a previous session's.
4982        for loss in judge_losses.into_iter().chain(vote_losses) {
4983            if recovered_keys.contains(&loss.seat) {
4984                continue;
4985            }
4986            self.state.quota.retain(|q| q.seat != loss.seat);
4987            self.state.quota.push(loss);
4988        }
4989
4990        // Recompute the verdict from the refreshed panel.
4991        self.state.tally = None;
4992        self.tally()?;
4993        Ok(self
4994            .state
4995            .tally
4996            .as_ref()
4997            .map(|t| t.met_quorum)
4998            .unwrap_or(false))
4999    }
5000
5001    // ---------------------------------------------------------------- graft
5002
5003    /// Port the strengths the judges attributed to the losing candidates onto
5004    /// the winner, once, between the tally and the review loop.
5005    ///
5006    /// Best-effort by construction: it returns nothing and fails nothing. A
5007    /// skip, a fixer that failed or hit quota, or a round that changed nothing
5008    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
5009    /// the run carries on as if the phase were off. See [`graft`] for the
5010    /// decisions and `AGENTS.md` for the invariants.
5011    ///
5012    /// The one exception is a rollback that could not be confirmed: the
5013    /// rejected commit may still be the winner's tip, so the run is blocked
5014    /// (resumable; the record stays `Running`, which makes a resume retry the
5015    /// restore) instead of reviewing, gating and merging it.
5016    async fn graft(&mut self) -> Result<()> {
5017        let Err(e) = self.graft_inner().await else {
5018            return Ok(());
5019        };
5020        if e.is::<crate::repo_lock::ResourceBlocked>() {
5021            return Err(e);
5022        }
5023        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024            self.state.status = RunStatus::Blocked;
5025            self.state.event("graft", format!("blocked: {e:#}"));
5026            self.state.save()?;
5027            return Err(e);
5028        }
5029        // An error after the `Running` record was saved may have left the
5030        // fixer's work in the tree: the record must not stay unsettled
5031        // (a later resume would reset past the review fixes), so confirm the
5032        // restore now, or block.
5033        if let Some(rec) = self.state.graft.clone()
5034            && !rec.outcome.settled()
5035        {
5036            if let Some(winner) = self.state.winner().cloned() {
5037                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038                    if re.is::<crate::repo_lock::ResourceBlocked>() {
5039                        return Err(re);
5040                    }
5041                    self.state.status = RunStatus::Blocked;
5042                    self.state.event("graft", format!("blocked: {re:#}"));
5043                    self.state.save()?;
5044                    return Err(re);
5045                }
5046            }
5047            if let Some(g) = self.state.graft.as_mut() {
5048                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049            }
5050        }
5051        self.state.event("graft", format!("skipped: {e:#}"));
5052        if let Err(e) = self.state.save() {
5053            tracing::warn!("save after graft: {e:#}");
5054        }
5055        Ok(())
5056    }
5057
5058    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
5059    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
5060    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061        let _coordination = self.coordinate(wt).await?;
5062        crate::recovery::restore_repair(wt, pre)
5063            .await
5064            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065    }
5066
5067    /// A skip is a run event, recorded once however often the node is re-entered.
5068    fn graft_skipped(&mut self, why: &str) {
5069        let message = format!("skipped: {why}");
5070        if !self
5071            .state
5072            .events
5073            .iter()
5074            .any(|e| e.node == "graft" && e.message == message)
5075        {
5076            self.state.event("graft", message);
5077        }
5078    }
5079
5080    async fn graft_inner(&mut self) -> Result<()> {
5081        // A round that was started and never finished (crash, kill): the
5082        // fixer may have left anything in the tree, so put the winner back.
5083        if let Some(rec) = self.state.graft.clone()
5084            && !rec.outcome.settled()
5085        {
5086            // Reviews exist only if the graft ended without its record being
5087            // settled (a swallowed error): the tree has moved on since, and
5088            // a reset to `pre` would discard the review fixes.
5089            if self.state.reviews.is_empty()
5090                && let Some(winner) = self.state.winner().cloned()
5091            {
5092                self.graft_restore(&winner.worktree, &rec.pre).await?;
5093            }
5094            if rec.outcome == GraftOutcome::WaitingForRestore {
5095                self.state.graft = None;
5096                self.state.save()?;
5097            } else {
5098                if let Some(g) = self.state.graft.as_mut() {
5099                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100                }
5101                self.state.event(
5102                    "graft",
5103                    "rolled back: the previous round was interrupted; the winner is untouched",
5104                );
5105                self.state.save()?;
5106                return Ok(());
5107            }
5108        }
5109        let plan = match graft::plan(&self.state) {
5110            Ok(plan) => plan,
5111            Err(skip) => {
5112                if skip != graft::Skip::AlreadyDone {
5113                    self.graft_skipped(skip.reason());
5114                    self.state.save()?;
5115                }
5116                return Ok(());
5117            }
5118        };
5119        let Some(winner) = self.state.winner().cloned() else {
5120            return Ok(());
5121        };
5122        let wt = winner.worktree.clone();
5123        if !git::is_clean(&wt).await? {
5124            self.graft_skipped("the winner's worktree has uncommitted changes");
5125            self.state.save()?;
5126            return Ok(());
5127        }
5128        let pre = git::rev_parse(&wt, "HEAD").await?;
5129        // Saved before the fixer is called: a stop from here on is repaired
5130        // by the branch above on the next entry.
5131        self.state.graft = Some(GraftRecord {
5132            pre: pre.clone(),
5133            post: None,
5134            lines: None,
5135            limit: None,
5136            agent: None,
5137            outcome: GraftOutcome::Running,
5138            requested: graft::requested(&plan),
5139            adoption: None,
5140            adoption_error: None,
5141        });
5142        self.state.event(
5143            "graft",
5144            format!(
5145                "porting strengths of candidate(s) {} onto {}",
5146                plan.strengths
5147                    .iter()
5148                    .map(|(l, _, _)| l.to_string())
5149                    .collect::<Vec<_>>()
5150                    .join(", "),
5151                plan.winner
5152            ),
5153        );
5154        self.state.save()?;
5155
5156        let fix_prompt = prompt::graft(
5157            &self.state.instruction,
5158            plan.winner,
5159            &plan.strengths,
5160            &self.state.config.graph.language,
5161        );
5162        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163        let sessions = self.state.config.graph.sessions;
5164        let artifacts = agent::artifacts_dir(&self.state.dir());
5165        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166        let (_, seat, out) = self
5167            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168                SeatJob {
5169                    prompt: fix_prompt.clone(),
5170                    spec,
5171                    seat,
5172                    cwd: wt.clone(),
5173                    timeout,
5174                    allow_write: true,
5175                    sessions,
5176                    artifacts: artifacts.clone(),
5177                    stem: "graft".to_owned(),
5178                    handover: None,
5179                }
5180            })
5181            .await;
5182        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183            // Preserve the need to restore earlier fallback edits without
5184            // confusing an unstarted seat with an interrupted invocation.
5185            self.state.graft.as_mut().expect("graft record").outcome =
5186                GraftOutcome::WaitingForRestore;
5187            return self.stop_for_resource();
5188        }
5189        let mut why: Option<String> = None;
5190        match &out {
5191            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192                why = Some("the fixer produced nothing usable".to_owned());
5193            }
5194            AgentOutcome::Ok(_) => {}
5195            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196            AgentOutcome::Quota(o) => {
5197                self.state.quota.push(QuotaLoss::from_report(
5198                    seat.key.clone(),
5199                    "graft",
5200                    o.quota.as_ref(),
5201                ));
5202                why = Some("rate limited (quota)".to_owned());
5203            }
5204            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205                why = Some(format!(
5206                    "the fixer failed: {}",
5207                    e.lines()
5208                        .next()
5209                        .unwrap_or("")
5210                        .chars()
5211                        .take(160)
5212                        .collect::<String>()
5213                ));
5214            }
5215        }
5216        let agent_id = seat.agent.clone();
5217        self.state.seats.insert(seat.key.clone(), seat);
5218
5219        let mut post = None;
5220        let mut lines = None;
5221        let _coordination = self.coordinate(&wt).await?;
5222        if why.is_none() {
5223            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224            if let Ok(r) = &rescued {
5225                self.state.note_withheld("graft", &r.withheld);
5226            }
5227            let head = git::rev_parse(&wt, "HEAD").await?;
5228            // What is measured must be what gets reviewed and gated: a failed
5229            // rescue, or anything still uncommitted (withheld lockfiles
5230            // aside, which are never part of the tree), rejects the graft.
5231            let dirty = match &rescued {
5232                Ok(r) => git::status_porcelain(&wt)
5233                    .await
5234                    .map(|st| {
5235                        st.lines().any(|l| {
5236                            !r.withheld
5237                                .iter()
5238                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239                        })
5240                    })
5241                    .unwrap_or(true),
5242                Err(_) => true,
5243            };
5244            if dirty {
5245                why = Some("the fixer left changes that could not be committed".to_owned());
5246            } else if head == pre {
5247                why = Some("the fixer changed nothing".to_owned());
5248            } else {
5249                lines = git::changed_lines(&wt, &pre, &head).await;
5250                post = Some(head);
5251            }
5252        }
5253        let (adoption, adoption_error) = match &out {
5254            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255            _ => (None, None),
5256        };
5257        let requested = graft::requested(&plan);
5258        let outcome = match why {
5259            None => {
5260                self.state.event(
5261                    "graft",
5262                    format!(
5263                        "applied: {}",
5264                        lines.map_or_else(
5265                            || "changed lines unavailable".to_owned(),
5266                            |n| format!("{n} changed lines")
5267                        )
5268                    ),
5269                );
5270                GraftOutcome::Applied
5271            }
5272            Some(why) => {
5273                // Not settled until the restore is confirmed: the record
5274                // stays `Running`, so a resume retries it.
5275                crate::recovery::restore_repair(&wt, &pre)
5276                    .await
5277                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278                self.state.event(
5279                    "graft",
5280                    format!("rolled back: {why}; the winner is untouched"),
5281                );
5282                GraftOutcome::RolledBack(why)
5283            }
5284        };
5285        self.state.graft = Some(GraftRecord {
5286            pre,
5287            post,
5288            lines,
5289            limit: None,
5290            agent: Some(agent_id),
5291            outcome,
5292            requested,
5293            adoption,
5294            adoption_error,
5295        });
5296        self.state.save()?;
5297        Ok(())
5298    }
5299
5300    // ----------------------------------------------------------------- fold
5301
5302    async fn fold_losers(&mut self) -> Result<()> {
5303        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304            return Ok(());
5305        };
5306        if !self.state.candidate_preservation.is_empty() {
5307            bail!("candidate preservation incomplete; refusing cleanup");
5308        }
5309        let repo = self.state.repo.clone();
5310        let _coordination = self.coordinate(&repo).await?;
5311        let mut folded = Vec::new();
5312        for i in 0..self.state.candidates.len() {
5313            let c = &self.state.candidates[i];
5314            if c.label == winner || c.folded {
5315                continue;
5316            }
5317            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318            remove_preserved_worktree(&repo, &wt).await?;
5319            if git::branch_exists(&repo, &branch).await?
5320                && !git::branch_delete(&repo, &branch).await?
5321            {
5322                bail!("candidate branch removal refused");
5323            }
5324            self.state.candidates[i].folded = true;
5325            self.state.save()?;
5326            folded.push(label.to_string());
5327        }
5328        // The judges are finished; their checkouts are pure cost from here.
5329        let root = self.state.worktree_root();
5330        for j in 1..=self.roles.judges.len() {
5331            let wt = root.join(format!("judge-{j}"));
5332            if wt.exists() {
5333                remove_preserved_worktree(&repo, &wt).await?;
5334            }
5335        }
5336        // The design-deliberation stage is finished by the time a tally
5337        // exists — same reasoning as the judges above.
5338        if self.state.config.graph.advise {
5339            for k in 1..=self.state.config.graph.advisors {
5340                let wt = root.join(format!("advisor-{k}"));
5341                if wt.exists() {
5342                    remove_preserved_worktree(&repo, &wt).await?;
5343                }
5344            }
5345        }
5346        if !folded.is_empty() {
5347            self.state
5348                .event("fold", format!("folded candidates {}", folded.join(", ")));
5349            self.state.save()?;
5350        }
5351        Ok(())
5352    }
5353
5354    // ------------------------------------------------------------ base sync
5355
5356    async fn hold_base_sync(&mut self) -> Result<bool> {
5357        self.hold_base_sync_as(None).await
5358    }
5359
5360    async fn hold_base_sync_kind(
5361        &mut self,
5362        kind: crate::base_sync_recovery::StopKind,
5363    ) -> Result<()> {
5364        self.hold_base_sync_as(Some(kind)).await?;
5365        Ok(())
5366    }
5367
5368    async fn hold_base_sync_as(
5369        &mut self,
5370        kind: Option<crate::base_sync_recovery::StopKind>,
5371    ) -> Result<bool> {
5372        if !self.state.base_sync_blocked() {
5373            self.state.base_sync_recovery.stop = None;
5374            return Ok(false);
5375        }
5376        let head = match self.state.winner() {
5377            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5378                .await
5379                .unwrap_or_default(),
5380            None => String::new(),
5381        };
5382        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5383            && !authorization.consumed
5384            && authorization.stop.head != head
5385        {
5386            authorization.refuse(format!(
5387                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5388                authorization.stop.head
5389            ));
5390        }
5391        self.state.record_base_sync_stop(&head);
5392        if let Some(kind) = kind {
5393            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5394            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5395                stop.kind = kind;
5396            }
5397        }
5398        self.state.save()?;
5399        Ok(true)
5400    }
5401
5402    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5403    /// anything verifies it.
5404    ///
5405    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5406    /// read whatever is checked out in the winner's worktree. Left alone that
5407    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5408    /// it when the run *branched* - and a run takes long enough that the base
5409    /// has usually moved by the time it gets here. A gate that ran there
5410    /// answers "green on the commit this run started from", not "green on
5411    /// what is about to land", and the difference showed up three times in
5412    /// one day as a green run whose merge would have reverted a file another
5413    /// pull request had already landed.
5414    ///
5415    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5416    /// `land::Step::Rebase` calls, rather than a second implementation of the
5417    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5418    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5419    /// doc warns about ("two answers to a question nobody notices until a
5420    /// diff is wrong").
5421    ///
5422    /// Foreign-checkout reviews instead merge the base into the owned shadow
5423    /// worktree, preserving the ancestry required for fast-forward delivery.
5424    /// A conflicting merge is aborted and held for operator resolution.
5425    ///
5426    /// A conflict is not the end of the road: the standing rebase is handed
5427    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5428    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5429    /// land). Once it finishes, review and the gate run as usual on the
5430    /// rebased tree, which is where a breakage the new base caused is caught
5431    /// by the ordinary gate-fix round. magi resolves nothing itself.
5432    ///
5433    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5434    /// the current durable recovery interval, bounds how often the base is
5435    /// synchronized during active recovery (lifetime attempts remain evidence).
5436    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5437    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5438    /// what was tried (rounds spent, paths still conflicted) and the branch
5439    /// and worktree stay exactly as they were - untouched, for a person to
5440    /// look at - which is also what makes re-entering this function
5441    /// afterwards a no-op instead of a second attempt at the same wall. A
5442    /// push failure ends the same way.
5443    async fn sync_to_base(&mut self) -> Result<()> {
5444        if self.state.status == RunStatus::AlreadyInBase {
5445            return Ok(());
5446        }
5447        self.hold_base_sync().await?;
5448        let conflicted = self
5449            .state
5450            .base_sync
5451            .as_ref()
5452            .is_some_and(|s| s.conflict.is_some());
5453        let Some(winner) = self.state.winner().cloned() else {
5454            return Ok(());
5455        };
5456
5457        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5458
5459        let repo = self.state.repo.clone();
5460        let remote = self.state.config.merge.remote.clone();
5461        let base_branch = self.state.base_branch.clone();
5462        let tracking = format!("{remote}/{base_branch}");
5463
5464        let fetched_base =
5465            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5466        // No network, or the remote never had this branch: the run already
5467        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5468        // otherwise), and one that got this far is not blocked by a fetch
5469        // that fails now. It just has no newer tip to compare against.
5470        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5471            return Ok(());
5472        };
5473
5474        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5475        self.state.ensure_base_episode();
5476        let mut pending = self
5477            .state
5478            .base_sync_recovery
5479            .operations
5480            .last()
5481            .filter(|o| !o.finished)
5482            .cloned();
5483        let scratch = self.state.dir().join("base-sync");
5484        if git::rebase_in_progress(&scratch).await {
5485            // Older resource stops predate operation reservations. Import the
5486            // standing Git operation, never retarget it to today's main.
5487            if pending.is_none()
5488                && self.state.base_sync_recovery.operations.is_empty()
5489                && !conflicted
5490                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5491                    .await
5492                    .ok()
5493                    .as_deref()
5494                    == Some(head.as_str())
5495                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5496                && git::is_clean(&winner.worktree).await?
5497            {
5498                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5499                if self.state.base_sync.is_none() {
5500                    self.state.base_sync = Some(BaseSync {
5501                        tip: saved_tip.clone(),
5502                        behind: 0,
5503                        attempts: 0,
5504                        conflict: None,
5505                        already_in: None,
5506                    });
5507                }
5508                self.state.reserve_base_operation(&head, &saved_tip);
5509                self.state.event(
5510                    "base-recovery",
5511                    "imported standing legacy rebase as a spent synchronization reservation",
5512                );
5513                self.state.save()?;
5514                pending = self.state.base_sync_recovery.operations.last().cloned();
5515            }
5516            let expected = pending.as_ref().map(|op| op.tip.as_str());
5517            if expected.is_none()
5518                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5519            {
5520                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5521                if self.state.base_sync.is_none() {
5522                    self.state.base_sync = Some(BaseSync {
5523                        tip: tip.clone(),
5524                        behind: 0,
5525                        attempts: 0,
5526                        conflict: None,
5527                        already_in: None,
5528                    });
5529                }
5530                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5531                self.state.event("land", why);
5532                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5533                    .await?;
5534                return Ok(());
5535            }
5536        }
5537        if let Some(op) = &pending {
5538            // A replay may add the pinned base's commits, never foreign work.
5539            let mut replay_proven =
5540                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5541            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5542                let wait = repository_wait(&self.state, None);
5543                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5544                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5545                    Ok(true) => {
5546                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5547                        self.state.save()?;
5548                        git::worktree_remove(&repo, &scratch).await.ok();
5549                        replay_proven = true;
5550                    }
5551                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5552                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5553                }
5554            }
5555            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5556                if self.state.foreign_checkout.is_some() {
5557                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5558                        .await
5559                        .ok()
5560                        .as_deref()
5561                        == Some(op.head.as_str())
5562                        && git::rev_parse(&repo, &format!("{head}^2"))
5563                            .await
5564                            .ok()
5565                            .as_deref()
5566                            == Some(op.tip.as_str());
5567                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5568                    .await
5569                    .is_ok_and(|m| m.is_empty())
5570                    && let Ok(added) =
5571                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5572                {
5573                    replay_proven = true;
5574                    for commit in added {
5575                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5576                            replay_proven = false;
5577                            break;
5578                        }
5579                    }
5580                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5581                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5582                }
5583            }
5584            // Never retarget an operation across a crash, even if main moved.
5585            // sync_to_head resets and cleans with -x, so even ignored untracked
5586            // files must stop recovery independently of the old tracked tree.
5587            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5588                || (head != op.head
5589                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5590                        .await?
5591                        .is_empty())
5592                || (!git::is_clean(&winner.worktree).await?
5593                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5594                        .await
5595                        .is_ok_and(|o| o.ok()))
5596                || !replay_proven
5597            {
5598                let why = format!(
5599                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5600                    op.id
5601                );
5602                let sync = self
5603                    .state
5604                    .base_sync
5605                    .as_mut()
5606                    .context("reserved synchronization has no lifetime accounting")?;
5607                sync.conflict = Some(why.clone());
5608                self.state.event("land", why);
5609                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5610                    .await?;
5611                return Ok(());
5612            }
5613            if head != op.head {
5614                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5615            }
5616            tip = op.tip.clone();
5617        }
5618        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5619        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5620        if pending.is_none()
5621            && !conflicted
5622            && fetched_base
5623            && self
5624                .state
5625                .base_sync_recovery
5626                .episodes
5627                .last()
5628                .is_some_and(|e| e.boundary.is_some())
5629            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5630            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5631            && owner.execution_runs().last() == Some(&self.state.id)
5632            && owner.status != crate::queue::TaskStatus::Done
5633            && !owner.fresh_start
5634            && owner.review_branch.is_none()
5635            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5636            && let Some(approval) = owner.resumption.as_ref()
5637            && approval.instruction == owner.instruction
5638            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5639            && git::is_clean(&winner.worktree).await?
5640            && self.state.renew_base_episode(&task, approval, &head, &tip)
5641        {
5642            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5643            self.state.save()?;
5644        }
5645
5646        // Before any rebase, and before a recorded conflict is honoured: a
5647        // branch whose change reached the base under other commit ids has
5648        // nothing to rebase and nothing to conflict with, and a run that
5649        // already stopped on that phantom conflict recovers here on resume.
5650        // `behind == 0` with head == tip is a branch the base has since taken
5651        // in whole, whether or not a conflict was ever recorded: the ancestry
5652        // proof must still run (`classify` ignores a head still on the start
5653        // commit).
5654        if (behind > 0 || conflicted || head == tip)
5655            && self
5656                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5657                .await?
5658        {
5659            return Ok(());
5660        }
5661        if conflicted && self.state.base_recovery_pending() {
5662            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5663            let owner =
5664                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5665            let owner_valid = owner.as_ref().is_ok_and(|t| {
5666                t.execution_runs().last() == Some(&self.state.id)
5667                    && t.status != crate::queue::TaskStatus::Done
5668                    && !t.fresh_start
5669                    && t.review_branch.is_none()
5670            });
5671            let invalidation = if !owner_valid {
5672                Some("latest task ownership changed")
5673            } else if !owner.as_ref().is_ok_and(|t| {
5674                authorization
5675                    .generation
5676                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5677            }) {
5678                Some("owner approval generation changed")
5679            } else if authorization.question.as_ref().is_some_and(|question| {
5680                !crate::base_recovery_control::decision_current(
5681                    &crate::queue::Queue::at(crate::run::home().join("queue")),
5682                    &self.state,
5683                    &authorization.task,
5684                    question,
5685                )
5686            }) {
5687                Some("recovery decision ownership, routing or stop evidence changed")
5688            } else if authorization.request.is_some()
5689                && !owner
5690                    .as_ref()
5691                    .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5692            {
5693                Some("owner decision changed")
5694            } else if let Some(scope) = &authorization.scope {
5695                if scope.tip != tip {
5696                    Some("fetched base changed")
5697                } else if !self
5698                    .state
5699                    .base_sync_recovery
5700                    .episodes
5701                    .last()
5702                    .is_some_and(|e| e.id == scope.episode)
5703                {
5704                    Some("active synchronization interval changed")
5705                } else if !owner
5706                    .as_ref()
5707                    .is_ok_and(|t| t.instruction == scope.instruction)
5708                {
5709                    Some("task instruction changed")
5710                } else {
5711                    None
5712                }
5713            } else {
5714                None
5715            };
5716            if let Some(reason) = invalidation {
5717                let authorization = self
5718                    .state
5719                    .base_sync_recovery
5720                    .authorizations
5721                    .last_mut()
5722                    .unwrap();
5723                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5724                self.state.save()?;
5725                return Ok(());
5726            }
5727            if !fetched_base
5728                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5729                || !git::is_clean(&winner.worktree).await?
5730                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5731            {
5732                return Ok(());
5733            }
5734        }
5735        let recovery = conflicted && self.state.consume_base_recovery(&head);
5736        if conflicted {
5737            // Consumption (including a refused changed HEAD) is durable before Git.
5738            self.state.save()?;
5739            if !recovery {
5740                return Ok(());
5741            }
5742        }
5743
5744        if behind == 0 {
5745            // A fixer-finished rebase moves the branch ref before the
5746            // winner's worktree is told (`sync_to_head` below). A run that
5747            // died in between resumes here with `behind == 0` and a tree still
5748            // holding the pre-rebase files, which review and the gate would
5749            // then read. That state is exactly: HEAD moved off the tip the
5750            // rebase started from, yet the tree is still identical to that
5751            // tip. A tree with edits of its own differs from it, so nothing
5752            // is thrown away.
5753            if let Some(from) = self
5754                .state
5755                .rebase_fixes
5756                .iter()
5757                .rev()
5758                .find_map(|r| r.from.clone())
5759                && from != head
5760                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5761                    .await
5762                    .is_ok_and(|o| o.ok())
5763            {
5764                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5765            }
5766            // An earlier attempt may have rebased the branch locally and died
5767            // before pushing it (only the fresh-rebase arm below pushes).
5768            // Publish it now, so the plain push at PR time is not refused as
5769            // a non-fast-forward. A run already holding a recorded conflict
5770            // never reaches here; that case is out of scope.
5771            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5772            if let Some(why) = &conflict {
5773                self.state.status = RunStatus::Blocked;
5774                self.state.event("land", why.clone());
5775            }
5776            self.state.base_sync = Some(BaseSync {
5777                tip,
5778                behind: 0,
5779                attempts,
5780                conflict,
5781                already_in: None,
5782            });
5783            if pending.is_some() {
5784                self.state.finish_base_operation(Some(&head));
5785            }
5786            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5787                .await?;
5788            self.state.save()?;
5789            return Ok(());
5790        }
5791
5792        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5793            let why = format!(
5794                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5795                 synchronization(s); synchronizing again would only race it",
5796                winner.branch
5797            );
5798            self.state.status = RunStatus::Blocked;
5799            self.state.base_sync = Some(BaseSync {
5800                tip,
5801                behind,
5802                attempts,
5803                conflict: Some(why.clone()),
5804                already_in: None,
5805            });
5806            self.state.event("land", why);
5807            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5808                .await?;
5809            self.state.save()?;
5810            return Ok(());
5811        }
5812
5813        if self.state.foreign_checkout.is_some() {
5814            // Delivery to the operator's checkout is fast-forward only. A
5815            // rebase here would discard its saved head from our ancestry.
5816            let wait = repository_wait(&self.state, None);
5817            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5818            if self.state.base_sync.is_none() {
5819                self.state.base_sync = Some(BaseSync {
5820                    tip: tip.clone(),
5821                    behind,
5822                    attempts: 0,
5823                    conflict: None,
5824                    already_in: None,
5825                });
5826            }
5827            self.state.reserve_base_operation(&head, &tip);
5828            self.state.save()?;
5829            let merged = git::git_raw(
5830                &winner.worktree,
5831                &[
5832                    "-c",
5833                    "core.hooksPath=/dev/null",
5834                    "merge",
5835                    "--no-ff",
5836                    "--no-autostash",
5837                    "--no-edit",
5838                    "-m",
5839                    &format!("Merge {tracking} before verifying"),
5840                    &tip,
5841                ],
5842            )
5843            .await?;
5844            let conflict = if merged.ok() {
5845                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5846                self.state.event(
5847                    "land",
5848                    format!("merged {tracking} into {} before verifying", winner.branch),
5849                );
5850                None
5851            } else {
5852                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5853                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5854                }
5855                let why = format!(
5856                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5857                    winner.branch,
5858                    merged.stderr.chars().take(600).collect::<String>()
5859                );
5860                self.state.status = RunStatus::Blocked;
5861                self.state.event("land", &why);
5862                Some(why)
5863            };
5864            let synced = conflict.is_none();
5865            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5866            self.state
5867                .finish_base_operation(synced.then_some(fresh.as_str()));
5868            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5869            self.state.base_sync = Some(BaseSync {
5870                tip,
5871                behind: if synced { 0 } else { behind },
5872                attempts,
5873                conflict,
5874                already_in: None,
5875            });
5876            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5877                .await?;
5878            self.state.save()?;
5879            return Ok(());
5880        }
5881
5882        self.state.event(
5883            "land",
5884            format!(
5885                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5886                winner.branch
5887            ),
5888        );
5889        self.state.save()?;
5890
5891        // The remote's copy of the branch, read now and only if the fetch
5892        // really succeeded (a stale tracking ref must never pin a lease). It is
5893        // pushed over after a rebase only when it is a commit this branch
5894        // already contains, by ancestry or by patch (an earlier rebase of ours
5895        // that never reached the remote): anything else is somebody else's work.
5896        let branch_tracking = format!("{remote}/{}", winner.branch);
5897        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5898        let remote_tip = if self.state.task_pr_branch().is_none()
5899            && matches!(&fetched_branch, Ok(o) if o.ok())
5900        {
5901            git::rev_parse(&repo, &branch_tracking).await.ok()
5902        } else {
5903            None
5904        };
5905        // A remote tip this branch does not contain is somebody else's work:
5906        // rebasing would leave a local tip that can never be pushed. Stop
5907        // before touching anything and say so.
5908        if let Some(theirs) = &remote_tip
5909            && !git::is_ancestor(&repo, theirs, &head).await
5910            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5911                .await
5912                .is_ok_and(|missing| missing.is_empty())
5913        {
5914            let why = format!(
5915                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5916                 them",
5917                short(theirs),
5918                winner.branch
5919            );
5920            self.state.status = RunStatus::Blocked;
5921            self.state.base_sync = Some(BaseSync {
5922                tip,
5923                behind,
5924                attempts,
5925                conflict: Some(why.clone()),
5926                already_in: None,
5927            });
5928            self.state.event("land", why);
5929            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5930                .await?;
5931            self.state.save()?;
5932            return Ok(());
5933        }
5934
5935        if self.state.base_sync.is_none() {
5936            self.state.base_sync = Some(BaseSync {
5937                tip: tip.clone(),
5938                behind,
5939                attempts: 0,
5940                conflict: None,
5941                already_in: None,
5942            });
5943        }
5944        self.state.reserve_base_operation(&head, &tip);
5945        self.state.save()?;
5946        let scratch = self.state.dir().join("base-sync");
5947        let rebased =
5948            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5949                .await
5950            {
5951                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5952                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5953                Err(e) => Err(e),
5954            };
5955        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5956        let stop_kind = match &rebased {
5957            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5958            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5959            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5960        };
5961        match rebased {
5962            Ok(None) => {
5963                // The branch ref moved, but a worktree that already had it
5964                // checked out (the winner's) was not told; sync its index and
5965                // files before anything reads them.
5966                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5967                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5968                self.state.finish_base_operation(Some(&fresh));
5969                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5970                let mut conflict = None;
5971                if let Some(pinned) = &remote_tip {
5972                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5973                    match pushed {
5974                        Ok(o) if o.ok() => self.state.event(
5975                            "land",
5976                            format!("pushed rebased {} to {remote}", winner.branch),
5977                        ),
5978                        Ok(o) => {
5979                            conflict = Some(format!(
5980                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5981                                winner.branch,
5982                                short(pinned),
5983                                o.stderr.chars().take(600).collect::<String>()
5984                            ));
5985                        }
5986                        Err(e) => {
5987                            conflict = Some(format!(
5988                                "rebased {} locally but could not push it: {e:#}",
5989                                winner.branch
5990                            ));
5991                        }
5992                    }
5993                }
5994                if let Some(why) = &conflict {
5995                    self.state.status = RunStatus::Blocked;
5996                    self.state.event("land", why.clone());
5997                }
5998                self.state.base_sync = Some(BaseSync {
5999                    tip: tip.clone(),
6000                    behind: 0,
6001                    attempts,
6002                    conflict,
6003                    already_in: None,
6004                });
6005                self.state
6006                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
6007            }
6008            Ok(Some(conflict)) => {
6009                self.state.finish_base_operation(None);
6010                let why = format!(
6011                    "{} conflicts with {tracking} and did not rebase: {}",
6012                    winner.branch,
6013                    conflict.chars().take(600).collect::<String>()
6014                );
6015                self.state.status = RunStatus::Blocked;
6016                self.state.base_sync = Some(BaseSync {
6017                    tip,
6018                    behind,
6019                    attempts,
6020                    conflict: Some(why.clone()),
6021                    already_in: None,
6022                });
6023                self.state.event("land", why);
6024            }
6025            // A refused Git write or a busy repository: nothing is wrong with
6026            // the branch. The scratch rebase stays standing and the run
6027            // resumes into it (`base_sync` must not claim a conflict).
6028            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6029                self.state.save()?;
6030                return Ok(());
6031            }
6032            Err(e) => {
6033                self.state.finish_base_operation(None);
6034                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6035                self.state.status = RunStatus::Blocked;
6036                self.state.base_sync = Some(BaseSync {
6037                    tip,
6038                    behind,
6039                    attempts,
6040                    conflict: Some(why.clone()),
6041                    already_in: None,
6042                });
6043                self.state.event("land", why);
6044            }
6045        }
6046        if !self.state.base_sync_blocked() {
6047            self.state.base_sync_recovery.stop = None;
6048        }
6049        self.hold_base_sync_kind(stop_kind).await?;
6050        self.state.save()?;
6051        Ok(())
6052    }
6053
6054    /// Push a branch an earlier attempt rebased locally but never published,
6055    /// pinned to the remote tip read right after a successful fetch. Returns
6056    /// the reason when the run must stop; `None` when there was nothing to do
6057    /// (no remote copy, the same tip, or a remote copy this branch already
6058    /// contains, which the PR-time push fast-forwards) or the push succeeded.
6059    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6060        // Task competitions publish only after the PR text gate, to their
6061        // public head; never resurrect a candidate branch on the remote.
6062        if self.state.task_pr_branch().is_some() {
6063            return None;
6064        }
6065        let repo = self.state.repo.clone();
6066        let remote = self.state.config.merge.remote.clone();
6067        let fetched = git::fetch(&repo, &remote, branch).await;
6068        if !matches!(&fetched, Ok(o) if o.ok()) {
6069            return None;
6070        }
6071        let branch_tracking = format!("{remote}/{branch}");
6072        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6073        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6074            return None;
6075        }
6076        if !crate::reconcile::origin_missing(&repo, head, &theirs)
6077            .await
6078            .is_ok_and(|missing| missing.is_empty())
6079        {
6080            return Some(format!(
6081                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6082                 them",
6083                short(&theirs)
6084            ));
6085        }
6086        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6087            Ok(o) if o.ok() => {
6088                self.state
6089                    .event("land", format!("pushed rebased {branch} to {remote}"));
6090                None
6091            }
6092            Ok(o) => Some(format!(
6093                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6094                 someone may have pushed): {}",
6095                short(&theirs),
6096                o.stderr.chars().take(600).collect::<String>()
6097            )),
6098            Err(e) => Some(format!(
6099                "{branch} is rebased locally but could not be pushed: {e:#}"
6100            )),
6101        }
6102    }
6103
6104    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6105    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6106    /// whether it did.
6107    ///
6108    /// Checked only when the base is ahead of the branch. A failing check is
6109    /// "not proven" - the ordinary rebase path then decides - never a reason to
6110    /// stop the run.
6111    ///
6112    /// The remote copy of the branch is held to the same standard as the local
6113    /// one: if it carries a tip this worktree does not, that tip must itself be
6114    /// proven in the base, or nothing is settled (a pull request would
6115    /// otherwise be closed over commits nobody checked). The pull request is
6116    /// closed *before* the terminal status is saved; if that fails for a
6117    /// reason other than a refusal (no network, a `gh` error) the run is left
6118    /// `Blocked` with the reason as its conflict, which a resume retries -
6119    /// the same recovery a phantom conflict gets.
6120    async fn settle_already_in(
6121        &mut self,
6122        branch: &str,
6123        tip: &str,
6124        head: &str,
6125        attempts: usize,
6126        behind: usize,
6127    ) -> Result<bool> {
6128        let repo = self.state.repo.clone();
6129        let remote = self.state.config.merge.remote.clone();
6130        let start = self.state.base_commit.clone();
6131        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6132            Ok(Some(e)) => e,
6133            Ok(None) => return Ok(false),
6134            Err(e) => {
6135                tracing::warn!("already-in-base check for {branch}: {e:#}");
6136                return Ok(false);
6137            }
6138        };
6139        let mut verified = vec![head.to_owned()];
6140        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6141        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6142        if matches!(&fetched, Ok(o) if o.ok())
6143            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6144            && theirs != head
6145        {
6146            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6147                Ok(Some(_)) => verified.push(theirs),
6148                _ => return Ok(false),
6149            }
6150        }
6151        let base_branch = self.state.base_branch.clone();
6152        let message = format!(
6153            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6154             land",
6155            evidence.names(),
6156            evidence.proof.as_str()
6157        );
6158        let closed =
6159            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6160                .await;
6161        match closed {
6162            Ok(Ok(url)) => self
6163                .state
6164                .event("land", format!("closed {url}: superseded on {base_branch}")),
6165            Ok(Err(why)) => self
6166                .state
6167                .event("land", format!("did not close a pull request: {why}")),
6168            Err(e) => {
6169                let why = format!(
6170                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6171                     not be closed ({e:#}); resume to retry"
6172                );
6173                self.state.status = RunStatus::Blocked;
6174                self.state.base_sync = Some(BaseSync {
6175                    tip: tip.to_owned(),
6176                    behind,
6177                    attempts,
6178                    conflict: Some(why.clone()),
6179                    already_in: None,
6180                });
6181                self.state.event("land", why);
6182                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6183                    .await?;
6184                self.state.save()?;
6185                return Ok(true);
6186            }
6187        }
6188        self.state.status = RunStatus::AlreadyInBase;
6189        self.state.base_sync_recovery.stop = None;
6190        self.state.base_sync = Some(BaseSync {
6191            tip: tip.to_owned(),
6192            behind,
6193            attempts,
6194            conflict: None,
6195            already_in: Some(evidence),
6196        });
6197        self.state.event("land", message);
6198        self.state.save()?;
6199        self.settle_questions();
6200        Ok(true)
6201    }
6202
6203    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6204    /// last landed the winner on, once it has run, else the commit the run
6205    /// branched from.
6206    ///
6207    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6208    /// and `vote` all happen before there is a winner to rebase, so they
6209    /// compare every candidate against the branch point on purpose, and a
6210    /// base that moves after they are already done cannot change an answer
6211    /// they already gave.
6212    fn landing_base(&self) -> String {
6213        self.state
6214            .base_sync
6215            .as_ref()
6216            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6217    }
6218
6219    // ------------------------------------------------------- operator fix
6220
6221    /// Route specific, already-recorded review findings to a fixer for a
6222    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6223    /// entry point.
6224    ///
6225    /// Distinct from `review_loop`'s own fix step in three ways: it never
6226    /// runs a reviewer wave, it never spends review-round budget, and what
6227    /// happened is recorded as an [`OperatorFixRequest`] appended to
6228    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6229    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6230    /// and vote must never be rewritten to look like a manufactured blocking
6231    /// verdict.
6232    ///
6233    /// Only meaningful once review has actually concluded: `Ready` (handed
6234    /// off with findings still open, or simply concluded clean while minor
6235    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6236    /// gate failed). Everything else is refused: a run still in progress
6237    /// should simply be resumed, and a `Merged` run's branch has already
6238    /// landed — reopening *this* run's own record cannot change that, so the
6239    /// answer there is a fresh `magi review <branch>`.
6240    ///
6241    /// A real commit here re-verifies through a fresh, ordinary review-only
6242    /// run on the same branch ([`Self::review`]) rather than reopening this
6243    /// run's own `review_loop`: once any round in this run's history went
6244    /// clean, `review_conclusion` treats that as permanent by design (the
6245    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6246    /// way to force one more genuine reviewer wave out of *this* run without
6247    /// either rewriting history or weakening that guarantee for every other
6248    /// caller. A review-only run costs nothing extra — no implementation, no
6249    /// judging, no vote — and exercises the exact same review → verify →
6250    /// gate → (human) merge path, unmodified.
6251    pub async fn fix_selected(
6252        &mut self,
6253        ids: &[String],
6254        reason: &str,
6255        allow_stale: bool,
6256    ) -> Result<()> {
6257        let reason = reason.trim();
6258        if reason.is_empty() {
6259            bail!("a fix request needs a reason — that is the operator's own record of why");
6260        }
6261        if ids.is_empty() {
6262            bail!("no finding id given");
6263        }
6264        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6265            bail!(
6266                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6267                 has already concluded — can be given a targeted fix. A run still \
6268                 in progress should simply be resumed; a `merged` run's branch has \
6269                 already landed, so its answer is a fresh `magi review <branch>`, \
6270                 not reopening this run's own record",
6271                self.state.id,
6272                self.state.status.as_str()
6273            );
6274        }
6275        // Reassignment shares this claim through its atomic task save. Check
6276        // the canonical run identity inside it, before preflight or any await.
6277        let _claim = FixClaim::acquire(&self.state.dir())?;
6278        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6279        let Some(winner) = self.state.winner().cloned() else {
6280            bail!("run {} has no winning candidate to fix", self.state.id);
6281        };
6282        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6283            bail!(
6284                "branch `{}` no longer exists; this run cannot be extended",
6285                winner.branch
6286            );
6287        }
6288        let home = crate::run::home();
6289        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6290            bail!(
6291                "run {} is currently being worked on by another magi process",
6292                self.state.id
6293            );
6294        }
6295        // Resolve every id before spending anything — an unknown id refuses
6296        // the whole request rather than silently dropping it — and dedup
6297        // while keeping the operator's own order.
6298        let mut seen = BTreeSet::new();
6299        let mut findings = Vec::new();
6300        let mut missing = Vec::new();
6301        for id in ids {
6302            if !seen.insert(id.clone()) {
6303                continue;
6304            }
6305            match self.state.finding(id) {
6306                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6307                    id: f.id.clone(),
6308                    severity: f.severity,
6309                    reviewer_vote: rec.vote,
6310                    round: round.round,
6311                    round_head: round.head.clone(),
6312                    reviewer: rec.reviewer,
6313                    agent: rec.agent.clone(),
6314                    file: f.file.clone(),
6315                    line: f.line,
6316                    title: f.title.clone(),
6317                    detail: f.detail.clone(),
6318                    outcome: OperatorFixOutcome::Pending,
6319                }),
6320                None => missing.push(id.clone()),
6321            }
6322        }
6323        if !missing.is_empty() {
6324            bail!(
6325                "unknown finding id(s): {}; nothing was changed",
6326                missing.join(", ")
6327            );
6328        }
6329
6330        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6331        let stale_details: Vec<(String, String)> = findings
6332            .iter()
6333            .filter(|f| f.round_head != head_at_request)
6334            .map(|f| (f.id.clone(), f.round_head.clone()))
6335            .collect();
6336        let stale = !stale_details.is_empty();
6337        if stale && !allow_stale {
6338            bail!(
6339                "the branch has moved since some finding(s) were raised — {} — now \
6340                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6341                stale_details
6342                    .iter()
6343                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6344                    .collect::<Vec<_>>()
6345                    .join(", "),
6346                short(&head_at_request)
6347            );
6348        }
6349
6350        let request = OperatorFixRequest {
6351            requested_at: Timestamp::now(),
6352            reason: reason.to_owned(),
6353            findings,
6354            head_at_request: head_at_request.clone(),
6355            allow_stale,
6356            stale,
6357            fix: None,
6358            result_head: None,
6359            follow_up_review_run: None,
6360        };
6361        self.state.event(
6362            "fix",
6363            format!(
6364                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6365                request.findings.len(),
6366                request
6367                    .findings
6368                    .iter()
6369                    .map(|f| f.id.as_str())
6370                    .collect::<Vec<_>>()
6371                    .join(", "),
6372            ),
6373        );
6374        // Recorded now, before any worktree work or the fixer call itself —
6375        // and re-saved at each checkpoint below: a crash at any point after
6376        // this (mid fixer call, mid follow-up review) must not lose the fact
6377        // that this was requested, for which findings, and why. Everything
6378        // past this point reads and writes through `request_index` rather
6379        // than a local variable, since `request` itself is moved here.
6380        self.state.operator_fixes.push(request);
6381        self.state.save()?;
6382        let request_index = self.state.operator_fixes.len() - 1;
6383
6384        // A fresh, dedicated worktree for this one call, never the winner's
6385        // own worktree in place: that one may already be gone (folded away),
6386        // and reusing it in place would leave the branch checked out there
6387        // when the follow-up review below tries to check it out again. Freed
6388        // immediately after, either way — but only once confirmed clean:
6389        // `worktree_remove` is a `git worktree remove --force`, which would
6390        // otherwise discard uncommitted work left there by the operator or
6391        // another process before this had a chance to even look at it.
6392        let repo = self.state.repo.clone();
6393        let coordination = self.coordinate(&repo).await?;
6394        if winner.worktree.exists() {
6395            // Lockfiles a rescue commit withheld stay untracked on purpose and
6396            // are already recorded; they are not the operator's work to protect.
6397            let dirty = git::git(
6398                &winner.worktree,
6399                &["status", "--porcelain", "--untracked-files=all"],
6400            )
6401            .await?;
6402            let only_withheld = dirty.lines().all(|l| {
6403                l.strip_prefix("?? ")
6404                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6405            });
6406            if !only_withheld {
6407                bail!(
6408                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6409                     discard them first",
6410                    winner.worktree.display()
6411                );
6412            }
6413            git::worktree_remove(&self.state.repo, &winner.worktree)
6414                .await
6415                .ok();
6416        }
6417        let fix_worktree = self.state.worktree_root().join("operator-fix");
6418        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6419        git::git(
6420            &self.state.repo,
6421            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6422        )
6423        .await
6424        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6425        if !git::is_clean(&fix_worktree).await? {
6426            git::worktree_remove(&self.state.repo, &fix_worktree)
6427                .await
6428                .ok();
6429            bail!(
6430                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6431                winner.branch
6432            );
6433        }
6434        let repo_for_setup = self.state.repo.clone();
6435        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6436            Ok(()) => {
6437                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6438            }
6439            Err(e) => Err(e),
6440        };
6441        if let Err(e) = setup {
6442            git::worktree_remove(&self.state.repo, &fix_worktree)
6443                .await
6444                .ok();
6445            return Err(e);
6446        }
6447
6448        // This explicit operator request retries availability independently
6449        // of execute(); the previous call has already saved and cleaned up.
6450        self.state.resource_block = None;
6451
6452        let run_id = self.state.id.clone();
6453        let prompts = self.state.config.prompts.clone();
6454        let language = self.state.config.graph.language.clone();
6455        let sessions = self.state.config.graph.sessions;
6456        let artifacts = agent::artifacts_dir(&self.state.dir());
6457        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6458            .findings
6459            .iter()
6460            .map(|f| Finding {
6461                id: f.id.clone(),
6462                severity: f.severity,
6463                file: f.file.clone(),
6464                line: f.line,
6465                title: f.title.clone(),
6466                detail: f.detail.clone(),
6467            })
6468            .collect();
6469        let fix_prompt = prompt::operator_fix_in(
6470            &self.state.instruction,
6471            &finding_list,
6472            reason,
6473            &stale_details,
6474            &head_at_request,
6475            &language,
6476            &self.state.config.graph.github_language,
6477        );
6478        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6479        drop(coordination);
6480        let (job, seat, out) = self
6481            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6482                prompt: fix_prompt.clone(),
6483                spec,
6484                seat,
6485                cwd: fix_worktree.clone(),
6486                timeout,
6487                allow_write: true,
6488                sessions,
6489                artifacts: artifacts.clone(),
6490                stem: "operator-fix".to_owned(),
6491                handover: None,
6492            })
6493            .await;
6494        let agent_id = seat.agent.clone();
6495
6496        let mut fix = FixRecord {
6497            agent: agent_id,
6498            addressed: Vec::new(),
6499            rejected: Vec::new(),
6500            notes: String::new(),
6501            committed: false,
6502            failed: None,
6503            duration_ms: 0,
6504            continuation: None,
6505        };
6506        let mut final_seat = seat.clone();
6507        match out {
6508            AgentOutcome::Ok(o) => {
6509                fix.duration_ms = o.duration_ms;
6510                let parsed = verdict::extract_json::<FixReport>(&o.text);
6511                let incomplete_reason = match &parsed {
6512                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6513                        "the reply parsed, but it reported a command whose own CLI \
6514                         never confirmed an exit status"
6515                            .to_owned(),
6516                    ),
6517                    Ok(_) => None,
6518                    Err(e) => Some(e.to_string()),
6519                };
6520                match incomplete_reason {
6521                    None => {
6522                        let report = parsed.expect("checked Ok above");
6523                        fix.addressed = report.addressed;
6524                        fix.rejected = report.rejected;
6525                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6526                    }
6527                    Some(reason) => {
6528                        let (resumed_seat, resolved, failure, cont) = self
6529                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6530                            .await;
6531                        fix.duration_ms += cont.cumulative_wait_ms;
6532                        fix.continuation = Some(cont);
6533                        final_seat = resumed_seat;
6534                        match resolved {
6535                            Some(report) => {
6536                                fix.addressed = report.addressed;
6537                                fix.rejected = report.rejected;
6538                                fix.notes =
6539                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6540                            }
6541                            None => fix.failed = failure,
6542                        }
6543                    }
6544                }
6545            }
6546            AgentOutcome::Dropped(o) => {
6547                fix.duration_ms = o.duration_ms;
6548                let why = o
6549                    .dropped
6550                    .as_ref()
6551                    .map(|d| d.why.as_str())
6552                    .unwrap_or("the CLI ended the stream without delivering its answer");
6553                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6554            }
6555            AgentOutcome::Quota(o) => {
6556                self.state.quota.push(QuotaLoss::from_report(
6557                    final_seat.key.clone(),
6558                    "fix",
6559                    o.quota.as_ref(),
6560                ));
6561                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6562            }
6563            AgentOutcome::Failed(e) => fix.failed = Some(e),
6564            AgentOutcome::ResourceBlocked(e) => {
6565                fix.failed = Some(format!("the fixer was not started: {e}"));
6566            }
6567        }
6568        // Even a deferred call must rescue edits, release the worktree and
6569        // finalize its record below so an explicit `magi fix` can retry.
6570        if fix.continuation.is_none() {
6571            fix.continuation = Some(ContinuationRecord::not_needed());
6572        }
6573        self.state.seats.insert(final_seat.key.clone(), final_seat);
6574
6575        let rescue_message = format!(
6576            "magi: operator-selected fix ({}) (uncommitted work)",
6577            self.state.operator_fixes[request_index]
6578                .findings
6579                .iter()
6580                .map(|f| f.id.as_str())
6581                .collect::<Vec<_>>()
6582                .join(", ")
6583        );
6584        let repo = self.state.repo.clone();
6585        let coordination = self.coordinate(&repo).await?;
6586        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6587            self.state.note_withheld("fix", &r.withheld);
6588        }
6589        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6590        fix.committed = after != head_at_request;
6591        git::worktree_remove(&self.state.repo, &fix_worktree)
6592            .await
6593            .ok();
6594        drop(coordination);
6595
6596        self.state.event(
6597            "fix",
6598            match &fix.failed {
6599                Some(reason) => format!(
6600                    "operator fix: adoption report was lost ({reason}); {}",
6601                    if fix.committed {
6602                        "committed"
6603                    } else {
6604                        "NO new commit"
6605                    }
6606                ),
6607                None => format!(
6608                    "operator fix: {} addressed, {} rejected, {}",
6609                    fix.addressed.len(),
6610                    fix.rejected.len(),
6611                    if fix.committed {
6612                        "committed"
6613                    } else {
6614                        "NO new commit"
6615                    }
6616                ),
6617            },
6618        );
6619
6620        // Every selected finding gets an outcome — never left `Pending` once
6621        // the fixer's own turn is over. A report that never came back at all
6622        // marks every one of them `Unreported`, not silently "not addressed":
6623        // quota, a dropped stream, or an exhausted continuation are gaps in
6624        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6625        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6626        for f in &mut self.state.operator_fixes[request_index].findings {
6627            f.outcome = if fix.failed.is_some() {
6628                OperatorFixOutcome::Unreported
6629            } else if fix.addressed.contains(&f.id) {
6630                OperatorFixOutcome::Addressed
6631            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6632                OperatorFixOutcome::Rejected { why: r.why.clone() }
6633            } else {
6634                OperatorFixOutcome::Unreported
6635            };
6636        }
6637
6638        let committed = fix.committed;
6639        if committed {
6640            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6641        }
6642        self.state.operator_fixes[request_index].fix = Some(fix);
6643        // Saved again now that the fixer's own outcome is final, on top of
6644        // the save right after the request was first pushed above.
6645        self.state.save()?;
6646        if self.state.resource_block.is_some() {
6647            return self.stop_for_resource();
6648        }
6649
6650        if committed {
6651            self.state.event(
6652                "fix",
6653                format!(
6654                    "operator fix committed {}; opening a follow-up review-only run",
6655                    short(&after)
6656                ),
6657            );
6658            // The operator asked for the fix, and the follow-up serves whatever
6659            // task the run it follows served.
6660            let origin =
6661                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6662            match Self::review(
6663                &self.state.repo,
6664                &winner.branch,
6665                self.state.config.clone(),
6666                origin,
6667            )
6668            .await
6669            {
6670                Ok(mut follow_up) => {
6671                    follow_up.state.event(
6672                        "start",
6673                        format!(
6674                            "requested by an operator fix on run {} for finding(s) {}",
6675                            self.state.id,
6676                            self.state.operator_fixes[request_index]
6677                                .findings
6678                                .iter()
6679                                .map(|f| f.id.as_str())
6680                                .collect::<Vec<_>>()
6681                                .join(", "),
6682                        ),
6683                    );
6684                    follow_up.state.save()?;
6685                    let follow_up_id = follow_up.state.id.clone();
6686                    // The follow-up is a run like any other: it belongs to the
6687                    // task of the run it follows, or to one filed for it.
6688                    let adopted = match crate::direct::adopt(
6689                        &follow_up.state,
6690                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6691                    ) {
6692                        Ok(a) => a,
6693                        Err(e) => {
6694                            // An ownerless run must not spend agent calls; it
6695                            // stays saved, and `magi run --resume` adopts it.
6696                            self.state.event(
6697                                "fix",
6698                                format!(
6699                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6700                                ),
6701                            );
6702                            self.state.save()?;
6703                            return Ok(());
6704                        }
6705                    };
6706                    let executed = follow_up.execute().await;
6707                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6708                    if let Some(a) = adopted {
6709                        a.finish(&follow_up.state, executed);
6710                    }
6711                    if let Some(e) = failure {
6712                        self.state.event(
6713                            "fix",
6714                            format!(
6715                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6716                            ),
6717                        );
6718                    }
6719                    self.state.operator_fixes[request_index].follow_up_review_run =
6720                        Some(follow_up_id);
6721                }
6722                Err(e) => {
6723                    self.state.event(
6724                        "fix",
6725                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6726                    );
6727                }
6728            }
6729            self.state.save()?;
6730        }
6731
6732        Ok(())
6733    }
6734
6735    // --------------------------------------------------------------- review
6736
6737    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6738    /// names, in order, each at most once, moving on only when the call
6739    /// advances (an error, a quota hit or nothing usable - the same decision
6740    /// point `agent::chain_advances` is for the other chained roles).
6741    ///
6742    /// Each agent gets its own seat from `Runner::seat`: the same agent
6743    /// continues its conversation (the winner's own implementer seat when it
6744    /// is the winner's author, now that the competition is over), another
6745    /// takes a fresh one so the full prompt is sent again. A handover is
6746    /// recorded under `node`, which is what makes the fallback stick for the
6747    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6748    /// is returned, so an exhausted chain reads exactly like a single failed
6749    /// fixer: one quota loss, the same wording, the same refund. An earlier
6750    /// attempt's edits are left in the tree and judged, with the final
6751    /// attempt's, by what git says afterwards.
6752    ///
6753    /// Returns the job that produced the outcome, for `continue_fix_report`.
6754    async fn ask_fixer(
6755        &mut self,
6756        winner: &Candidate,
6757        node: &'static str,
6758        round: Option<usize>,
6759        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6760    ) -> (SeatJob, SeatState, AgentOutcome) {
6761        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6762        self.ask_fixer_among(attempts, node, round, None, build)
6763            .await
6764    }
6765
6766    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6767    /// worktree and a commit: before the next agent is asked after one that
6768    /// did not satisfy the call, the worktree is put back on that commit, so
6769    /// a failed agent's partial edits are not carried into the next one.
6770    async fn ask_fixer_among(
6771        &mut self,
6772        attempts: Vec<(AgentSpec, String)>,
6773        node: &'static str,
6774        round: Option<usize>,
6775        restore: Option<(&Path, &str)>,
6776        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6777    ) -> (SeatJob, SeatState, AgentOutcome) {
6778        let run_id = self.state.id.clone();
6779        let prompts = self.state.config.prompts.clone();
6780        let cache = self.state.config.cache_dir();
6781        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6782        let mut last = None;
6783        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6784            let seat = self.seat(&key, &spec.id);
6785            let mut job = build(spec.clone(), seat);
6786            if i > 0 {
6787                job.stem = format!("{}-{}", job.stem, spec.id);
6788            }
6789            let ctx = WaveCtx {
6790                carry_seats: false,
6791                pauses: Some((&self.pause, &self.interrupt)),
6792                run: &run_id,
6793                node,
6794                prompts: &prompts,
6795                cache: cache.as_deref(),
6796                round,
6797            };
6798            let (seat, out) =
6799                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6800            // A resource block never advances the chain: the chosen agent
6801            // was not asked, so it must be the one asked again on resume.
6802            let advances = match &out {
6803                AgentOutcome::Ok(o) => agent::output_advances(o),
6804                AgentOutcome::ResourceBlocked(_) => false,
6805                _ => true,
6806            };
6807            if let Some(next) = ids.get(i + 1)
6808                && advances
6809            {
6810                self.state.seats.insert(seat.key.clone(), seat.clone());
6811                let class =
6812                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6813                let reason = match &out {
6814                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6815                    other => fail_reason(other),
6816                };
6817                // A failed agent's edits must not reach the next one, and a
6818                // restore that did not work ends the chain: the call fails
6819                // and the caller's own (confirmed) restore decides.
6820                if let Some((wt, rev)) = restore
6821                    && let Err(e) = self.graft_restore(wt, rev).await
6822                {
6823                    let why =
6824                        format!("could not restore the worktree before the next fixer: {e:#}");
6825                    tracing::warn!("{why}");
6826                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6827                        AgentOutcome::ResourceBlocked(why)
6828                    } else {
6829                        AgentOutcome::Failed(why)
6830                    };
6831                    last = Some((job, seat, out));
6832                    break;
6833                }
6834                record_handover(
6835                    &mut self.state,
6836                    node,
6837                    &seat.key,
6838                    &spec.id,
6839                    next,
6840                    &class,
6841                    &reason,
6842                );
6843                continue;
6844            }
6845            last = Some((job, seat, out));
6846            break;
6847        }
6848        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6849        if let AgentOutcome::Quota(output) = &out
6850            && node != "graft"
6851        {
6852            // Graft is best effort: its caller records the loss and restores
6853            // the pre-graft tree, then proceeds to review and gate.
6854            // Every fallback was tried. Machine outages cannot be repaired by
6855            // spending a fix round; use each caller's resumable resource path.
6856            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6857            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6858            self.state.quota.push(loss);
6859            self.state.seats.insert(seat.key.clone(), seat.clone());
6860            self.state.status = RunStatus::Blocked;
6861            return (
6862                job,
6863                seat,
6864                AgentOutcome::ResourceBlocked(
6865                    "fixer unavailable; restore agent availability and resume".into(),
6866                ),
6867            );
6868        }
6869        (job, seat, out)
6870    }
6871
6872    /// The one repair a verification-only failure earns (see
6873    /// [`crate::recovery`]), before the first review round of a run that took
6874    /// a branch over.
6875    ///
6876    /// Saved as started *before* the fixer is called, so a resume never
6877    /// repairs twice; a start found on re-entry is rolled back to the commit
6878    /// it began from. A failed or unusable fixer leaves the tree as it was and
6879    /// the review goes on over it, so the first round's own verification
6880    /// reports the failure again. The failure is never carried into a round as
6881    /// a result: verification is re-run on whatever commit this leaves.
6882    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6883        use crate::recovery::{Mode, Repair};
6884        if !self.state.reviews.is_empty() {
6885            return Ok(());
6886        }
6887        let Some(mut rec) = self.state.recovery.clone() else {
6888            return Ok(());
6889        };
6890        if rec.mode != Mode::VerifyRepair {
6891            return Ok(());
6892        }
6893        let wt = winner.worktree.clone();
6894        let mut head = git::rev_parse(&wt, "HEAD").await?;
6895        match rec.repair.clone() {
6896            Repair::Done { .. } => return Ok(()),
6897            Repair::WaitingForRestore { before } => {
6898                self.graft_restore(&wt, &before).await?;
6899                head = before;
6900                rec.repair = Repair::NotStarted;
6901                self.state.recovery = Some(rec.clone());
6902                self.state.save()?;
6903            }
6904            Repair::Started { before } => {
6905                // Interrupted: the unfinished edits are not trusted.
6906                // An unconfirmed rollback blocks the run with `Started` kept,
6907                // so a resume retries it; the tree is never reviewed as is.
6908                if let Err(e) = self.graft_restore(&wt, &before).await {
6909                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6910                        return Err(e);
6911                    }
6912                    return self
6913                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6914                        .await;
6915                }
6916                rec.repair = Repair::Done {
6917                    before: before.clone(),
6918                    after: before,
6919                    committed: false,
6920                    failed: Some("interrupted; rolled back".to_owned()),
6921                };
6922                self.state
6923                    .event("recovery", "the repair was interrupted and rolled back");
6924                self.state.recovery = Some(rec);
6925                self.state.save()?;
6926                return Ok(());
6927            }
6928            Repair::NotStarted => {}
6929        }
6930        if crate::recovery::recheck(&mut rec, &head) {
6931            self.state.event("recovery", rec.summary());
6932            self.state.recovery = Some(rec);
6933            self.state.save()?;
6934            return Ok(());
6935        }
6936        rec.repair = Repair::Started {
6937            before: head.clone(),
6938        };
6939        self.state.recovery = Some(rec.clone());
6940        self.state.save()?;
6941
6942        let language = self.state.config.graph.language.clone();
6943        let sessions = self.state.config.graph.sessions;
6944        let artifacts = agent::artifacts_dir(&self.state.dir());
6945        let fix_prompt = prompt::recovery_fix(
6946            &self.state.instruction,
6947            &rec,
6948            self.state.config.graph.review_rounds,
6949            &language,
6950        );
6951        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6952        self.state.event(
6953            "recovery",
6954            format!(
6955                "repair: the fixer starts from the failed verification on {}",
6956                short(&head)
6957            ),
6958        );
6959        let (_job, seat, out) = self
6960            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6961                prompt: fix_prompt.clone(),
6962                spec,
6963                seat,
6964                cwd: wt.clone(),
6965                timeout,
6966                allow_write: true,
6967                sessions,
6968                artifacts: artifacts.clone(),
6969                stem: "fix-recovery".to_owned(),
6970                handover: None,
6971            })
6972            .await;
6973        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6974            rec.repair = Repair::WaitingForRestore { before: head };
6975            self.state.recovery = Some(rec);
6976            return self.stop_for_resource();
6977        }
6978        let failed = match &out {
6979            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6980            AgentOutcome::Quota(o) => {
6981                self.state.quota.push(QuotaLoss::from_report(
6982                    seat.key.clone(),
6983                    "fix",
6984                    o.quota.as_ref(),
6985                ));
6986                Some("rate limited (quota); the fixer could not run".to_owned())
6987            }
6988            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6989            AgentOutcome::Failed(e) => Some(e.clone()),
6990        };
6991        self.state.seats.insert(seat.key.clone(), seat);
6992        let coordination = self.coordinate(&wt).await?;
6993        let failed = match failed {
6994            // The agent's partial edits are not kept.
6995            Some(why) => {
6996                if let Err(e) = git::restore_to(&wt, &head).await {
6997                    return self
6998                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6999                        .await;
7000                }
7001                Some(why)
7002            }
7003            None => {
7004                if let Ok(r) = git::rescue_commit(
7005                    &wt,
7006                    "magi: recovery repair of the failed verification (uncommitted work)",
7007                )
7008                .await
7009                {
7010                    self.state.note_withheld("fix", &r.withheld);
7011                }
7012                None
7013            }
7014        };
7015        let after = git::rev_parse(&wt, "HEAD").await?;
7016        drop(coordination);
7017        self.state.event(
7018            "recovery",
7019            match &failed {
7020                Some(why) => format!("repair: nothing kept ({why})"),
7021                None if after != head => {
7022                    format!("repair: committed {}", short(&after))
7023                }
7024                None => "repair: the fixer made no commit".to_owned(),
7025            },
7026        );
7027        rec.repair = Repair::Done {
7028            before: head.clone(),
7029            committed: after != head,
7030            after,
7031            failed,
7032        };
7033        self.state.recovery = Some(rec);
7034        self.state.save()?;
7035        Ok(())
7036    }
7037
7038    /// A repair whose rollback could not be confirmed: the run is blocked and
7039    /// resumable, and the unconfirmed state is never reviewed.
7040    async fn stop_recovery(
7041        &mut self,
7042        mut rec: crate::recovery::RecoveryRecord,
7043        head: &str,
7044        why: String,
7045    ) -> Result<()> {
7046        // `Started` stays recorded: a resume retries the restore.
7047        rec.repair = crate::recovery::Repair::Started {
7048            before: head.to_owned(),
7049        };
7050        self.state.recovery = Some(rec);
7051        self.state
7052            .event("recovery", format!("repair could not be undone: {why}"));
7053        self.state.status = RunStatus::Blocked;
7054        self.state.save()?;
7055        bail!("recovery repair could not be rolled back: {why}")
7056    }
7057
7058    async fn review_loop(&mut self) -> Result<()> {
7059        if self.hold_base_sync().await? {
7060            return Ok(());
7061        }
7062        loop {
7063            if self
7064                .state
7065                .base_sync
7066                .as_ref()
7067                .is_some_and(|s| s.conflict.is_some())
7068            {
7069                return Ok(());
7070            }
7071            if self.catchup_repairable() {
7072                let record = self.state.reviews.last().expect("repairable round").clone();
7073                let winner = self
7074                    .state
7075                    .winner()
7076                    .cloned()
7077                    .context("catch-up repair has no winner")?;
7078                if !self.fix_review_round(&winner, record).await? {
7079                    return Ok(());
7080                }
7081            }
7082            self.review_loop_pass().await?;
7083            if !self.catchup_repairable() {
7084                return Ok(());
7085            }
7086        }
7087    }
7088
7089    /// Derive an owed catch-up repair from durable round evidence, including
7090    /// reentry after a crash before the fixer was dispatched.
7091    fn catchup_repairable(&self) -> bool {
7092        self.state.reviews.len() < self.state.config.graph.review_rounds
7093            && self.state.reviews.last().is_some_and(|r| {
7094                !r.clean
7095                    && r.blocking == 0
7096                    && r.fix.is_none()
7097                    && r.e2e_status() == E2eStatus::Failed
7098            })
7099    }
7100
7101    async fn review_loop_pass(&mut self) -> Result<()> {
7102        // A base that would not rebase is a person's decision, not a review
7103        // round: nothing here would change the answer, and reviewers and a
7104        // fixer would be spending real budget on a tree that cannot land
7105        // regardless of what they find.
7106        if self
7107            .state
7108            .base_sync
7109            .as_ref()
7110            .is_some_and(|s| s.conflict.is_some())
7111        {
7112            return Ok(());
7113        }
7114        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7115        // agent files with `magi task add` name the run that paid for it. The
7116        // prompt overlay is cloned alongside it because the waves borrow it
7117        // while `self` is mutably borrowed by the node's own bookkeeping.
7118        let run_id = self.state.id.clone();
7119        let prompts = self.state.config.prompts.clone();
7120        let Some(winner) = self.state.winner().cloned() else {
7121            return Ok(());
7122        };
7123        let granted_review = self
7124            .state
7125            .repair_grants
7126            .last()
7127            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7128            .cloned();
7129        let fresh_review = granted_review
7130            .as_ref()
7131            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7132        let max_rounds = granted_review
7133            .as_ref()
7134            .map_or(self.state.config.graph.review_rounds, |g| {
7135                g.review_start.saturating_add(1)
7136            });
7137        // A clean round, an exhausted round budget, or a stalled tree (see
7138        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7139        // they are recorded — recomputed here, not read off `status`, so a
7140        // reentry into a run that already stopped restates the identical
7141        // verdict instead of silently handing back whatever an earlier node
7142        // in this same walk clobbered `status` to (a solo-candidate
7143        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7144        // below runs an empty range once the budget is spent, and would
7145        // otherwise fall through without touching `status` at all.
7146        if let Some(status) =
7147            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7148        {
7149            // A reentry after a crash between the last round's save and
7150            // `stop_reviewing` reaches the hand-off here, not there.
7151            if status == RunStatus::Gating {
7152                self.record_contested_handoff();
7153            }
7154            self.state.status = status;
7155            self.state.save()?;
7156            return Ok(());
7157        }
7158        self.state.status = RunStatus::Reviewing;
7159        // A last recorded round whose own verification never resolved
7160        // (`ResourceBlocked` — the shared build cache, not the patch) is
7161        // never a concluded round, whatever the round budget says: starting
7162        // a fresh round on top of it would spend a whole new reviewer wave
7163        // re-reading an unchanged patch instead of just retrying the one
7164        // check that actually needs it, and once the budget is spent the
7165        // loop below has nothing left to do at all (its range is empty).
7166        // Retry that check directly instead, exactly the same retry
7167        // `stop_reviewing` already does for its own catch-up case.
7168        if !fresh_review
7169            && self.state.reviews.last().is_some_and(|r| {
7170                r.e2e_status() == E2eStatus::ResourceBlocked
7171                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7172            })
7173        {
7174            let shell = self.state.config.shell();
7175            return self
7176                .stop_unresolved_review(
7177                    "the last round's own verification never resolved",
7178                    &shell,
7179                    &winner.worktree,
7180                )
7181                .await;
7182        }
7183
7184        if granted_review.is_none() {
7185            self.recovery_repair(&winner).await?;
7186        }
7187        if self.resource_halt()? {
7188            return Ok(());
7189        }
7190
7191        let repo = self.state.repo.clone();
7192        let root = self.state.worktree_root();
7193        let language = self.state.config.graph.language.clone();
7194        let sessions = self.state.config.graph.sessions;
7195        let artifacts = self.merge_repair_artifacts();
7196        let base = self.landing_base();
7197        let base_short = short(&base);
7198        let reviewers = self.roles.reviewers.clone();
7199        let shell = self.state.config.shell();
7200
7201        for round in (self.state.reviews.len() + 1)..=max_rounds {
7202            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7203            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7204            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7205            // The prior round's own record, already persisted — never a
7206            // hand-carried variable of just its failing output: that is
7207            // exactly what let a round's e2e result drift out of sync with
7208            // which commit it was actually about (see `SCHEMA`'s doc for
7209            // schema 8). Judged against `head`, the commit reviewers are
7210            // about to look at now, so the summary always reads as "an
7211            // earlier head" here — this round's own patch has not been
7212            // checked yet.
7213            let prev_verification = self
7214                .state
7215                .reviews
7216                .last()
7217                .and_then(|r| r.verification_summary(&head));
7218            // Only the first round of a run that took a branch over: later
7219            // rounds have this run's own history.
7220            let prior_section = match (&self.state.recovery, round) {
7221                (Some(rec), 1) if self.state.reviews.is_empty() => {
7222                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7223                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7224                            Some(crate::recovery::head_of(
7225                                &git::diff(&winner.worktree, reviewed, "HEAD")
7226                                    .await
7227                                    .unwrap_or_default(),
7228                                crate::recovery::SINCE_DIFF_MAX,
7229                            ))
7230                        }
7231                        _ => None,
7232                    };
7233                    Some(prompt::recovery_review(rec, since.as_deref()))
7234                }
7235                _ => None,
7236            };
7237
7238            // Each reviewer gets its own detached checkout of exactly this
7239            // commit: nobody can perturb the winner's tree, and the fixer can
7240            // keep working without racing a reviewer.
7241            let mut jobs = Vec::new();
7242            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7243                self.coordinate(&repo).await?
7244            } else {
7245                None
7246            };
7247            for (r, spec) in reviewers.iter().cloned().enumerate() {
7248                let wt = root.join(format!("review-{}", r + 1));
7249                // `clean -fdx` in a reset wipes setup's products, so every
7250                // round sets the seat up again.
7251                if wt.exists() {
7252                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7253                } else {
7254                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7255                        None
7256                    } else {
7257                        self.coordinate(&repo).await?
7258                    };
7259                    git::worktree_add_detached(&repo, &wt, &head).await?;
7260                }
7261                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7262                    None
7263                } else {
7264                    self.coordinate(&repo).await?
7265                };
7266                self.setup_seat_worktree(&repo, &wt, false).await?;
7267                let seat_key = format!("review-{}", r + 1);
7268                // The seat starts the round on whoever answered it last, not
7269                // on the agent the spec names, so a failure is not re-paid.
7270                let spec = pick_start_spec(
7271                    &self.roles.reviewer_roster,
7272                    spec,
7273                    self.state.seat_history.get(&seat_key),
7274                );
7275                let seat = self.seat(&seat_key, &spec.id);
7276                jobs.push(SeatJob {
7277                    prompt: prompt::review(&prompt::ReviewCtx {
7278                        instruction: &self.state.instruction,
7279                        branch: &winner.branch,
7280                        base_short: &base_short,
7281                        stat: &stat,
7282                        patch: &patch,
7283                        verification: prev_verification.as_ref(),
7284                        prior: prior_section.as_deref(),
7285                        reviewers: reviewers.len(),
7286                        round,
7287                        rounds: max_rounds,
7288                        // A review-only run has no rankings, so nothing
7289                        // competed for this patch and the reviewer is told so.
7290                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7291                        lens: Lens::for_seat(r),
7292                        language: &language,
7293                        github_language: &self.state.config.graph.github_language,
7294                    }),
7295                    spec,
7296                    seat,
7297                    cwd: wt,
7298                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7299                    allow_write: false,
7300                    sessions,
7301                    artifacts: artifacts.clone(),
7302                    stem: format!("review-{round}-{}", r + 1),
7303                    handover: None,
7304                });
7305            }
7306
7307            drop(coordination);
7308            self.state.event(
7309                "review",
7310                format!(
7311                    "round {round}: {} reviewers on {}",
7312                    jobs.len(),
7313                    short(&head)
7314                ),
7315            );
7316            let mut quota_losses = Vec::new();
7317            let review_retries = self.state.config.graph.retries;
7318            let review_cache = self.state.config.cache_dir();
7319            let ctx = WaveCtx {
7320                carry_seats: true,
7321                pauses: Some((&self.pause, &self.interrupt)),
7322                run: &run_id,
7323                node: "review",
7324                prompts: &prompts,
7325                cache: review_cache.as_deref(),
7326                round: Some(round),
7327            };
7328            let results = ask_json_wave::<Review>(
7329                jobs,
7330                Arc::clone(&self.sem),
7331                review_retries,
7332                &self.roles.reviewer_roster,
7333                &ctx,
7334                &mut quota_losses,
7335                &mut self.state,
7336                &|_: &Review| Ok(()),
7337            )
7338            .await;
7339            if self.state.resource_block.is_some() {
7340                return self.stop_for_resource();
7341            }
7342            // Counted before the move below: how many of *this* round's
7343            // reviewer seats were lost to their own rate limit, as opposed to
7344            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7345            let round_quota_missing = quota_losses.len();
7346            self.state.quota.extend(quota_losses);
7347
7348            let mut records = Vec::new();
7349            let mut all_findings = Vec::new();
7350            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7351                let agent_id = seat.agent.clone();
7352                self.state.seats.insert(seat.key.clone(), seat);
7353                let mut record = ReviewRecord {
7354                    reviewer: r + 1,
7355                    agent: agent_id,
7356                    summary: String::new(),
7357                    findings: Vec::new(),
7358                    vote: None,
7359                    failed: None,
7360                    duration_ms: 0,
7361                    // Set for both outcomes: `failed: Some(_)` with
7362                    // `attempts > 0` is a seat every retry still lost, not a
7363                    // recovered one — only `failed: None` with `attempts > 0`
7364                    // reads as "answered after a nudge" (see this field's own
7365                    // doc).
7366                    attempts,
7367                };
7368                match res {
7369                    Ok((review, out)) => {
7370                        // Sanitized here, at the point every other piece of
7371                        // agent prose in this file is (candidate summaries,
7372                        // deliberation turns, vote reasons): a reviewer's own
7373                        // words are the one thing about it that could name
7374                        // it, and reconsideration below broadcasts this same
7375                        // summary and these same findings to every other
7376                        // seat on the panel.
7377                        record.summary =
7378                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7379                        record.vote = Some(review.vote);
7380                        record.duration_ms = out.duration_ms;
7381                        for (n, mut f) in review.findings.into_iter().enumerate() {
7382                            // ids are magi's, never the agent's: the fixer's
7383                            // adoption report is keyed by them.
7384                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7385                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7386                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7387                            // `file` is agent-supplied prose too, never
7388                            // checked against the real tree — the same
7389                            // exposure `title`/`detail` above have, just in
7390                            // a field easy to forget because it looks like a
7391                            // path rather than free text.
7392                            f.file = f
7393                                .file
7394                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7395                            all_findings.push(f.clone());
7396                            record.findings.push(f);
7397                        }
7398                        self.state.event(
7399                            "review",
7400                            format!(
7401                                "round {round}: reviewer {} voted {} with {} finding(s)",
7402                                r + 1,
7403                                review.vote.label(),
7404                                record.findings.len()
7405                            ),
7406                        );
7407                    }
7408                    Err(e) => {
7409                        record.failed = Some(e.to_string());
7410                        self.state.event(
7411                            "review",
7412                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7413                        );
7414                    }
7415                }
7416                records.push(record);
7417            }
7418
7419            // Tally the round's votes and, if they split, spend the one
7420            // round of reconsideration the split -> deliberate -> revote
7421            // shape `judge`/`vote` use for the panel, sized down to what a
7422            // read-only review round can afford: one round, and a revote
7423            // rather than an argument, because the panel already wrote its
7424            // reasoning down as findings the first time around.
7425            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7426            let vote_split =
7427                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7428            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7429            if vote_split {
7430                self.state.event(
7431                    "review",
7432                    format!(
7433                        "round {round}: votes split ({}) — one round of reconsideration",
7434                        initial_votes
7435                            .iter()
7436                            .map(|v| v.label())
7437                            .collect::<Vec<_>>()
7438                            .join(", ")
7439                    ),
7440                );
7441                // Seats read every seat's findings and votes, still numbered
7442                // and never named — the same anonymity `review` itself keeps.
7443                let panel: Vec<ReviewSeatReport<'_>> = records
7444                    .iter()
7445                    .filter_map(|r| {
7446                        r.vote.map(|vote| ReviewSeatReport {
7447                            reviewer: r.reviewer,
7448                            vote,
7449                            summary: &r.summary,
7450                            findings: &r.findings,
7451                        })
7452                    })
7453                    .collect();
7454
7455                let mut jobs = Vec::new();
7456                let mut seats_at = Vec::new();
7457                for (r, spec) in reviewers.iter().cloned().enumerate() {
7458                    // A seat with no initial vote has nothing to reconsider
7459                    // from and stays absent, the same as it stayed absent
7460                    // from `panel` above.
7461                    if records[r].vote.is_none() {
7462                        continue;
7463                    }
7464                    let wt = root.join(format!("review-{}", r + 1));
7465                    let seat_key = format!("review-{}", r + 1);
7466                    let spec = self.occupant(&seat_key, spec);
7467                    let seat = self.seat(&seat_key, &spec.id);
7468                    // A seat with no live session has already forgotten the
7469                    // initial review's prompt — restate the patch it is
7470                    // voting on, the same as `deliberate`/`vote` do for a
7471                    // judge in the same position.
7472                    // The panel already carries this seat's own review and
7473                    // vote, so restating the patch makes the prompt whole for
7474                    // a seat handed to another agent.
7475                    let build = |with_patch: bool| {
7476                        prompt::review_reconsider(&ReviewReconsiderCtx {
7477                            instruction: &self.state.instruction,
7478                            reviewer: r + 1,
7479                            lens: Lens::for_seat(r),
7480                            panel: &panel,
7481                            patch: with_patch.then_some(ReviewPatch {
7482                                branch: &winner.branch,
7483                                base_short: &base_short,
7484                                stat: &stat,
7485                                patch: &patch,
7486                            }),
7487                            round,
7488                            rounds: max_rounds,
7489                            language: &language,
7490                        })
7491                    };
7492                    let full = build(true);
7493                    let prompt = if has_context(&spec, &seat, sessions) {
7494                        build(false)
7495                    } else {
7496                        full.clone()
7497                    };
7498                    jobs.push(SeatJob {
7499                        prompt,
7500                        spec,
7501                        seat,
7502                        cwd: wt,
7503                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7504                        allow_write: false,
7505                        sessions,
7506                        artifacts: artifacts.clone(),
7507                        stem: format!("review-{round}-reconsider-{}", r + 1),
7508                        handover: Some(full),
7509                    });
7510                    seats_at.push(r);
7511                }
7512
7513                let mut recon_quota_losses = Vec::new();
7514                let recon_cache = self.state.config.cache_dir();
7515                let recon_ctx = WaveCtx {
7516                    carry_seats: true,
7517                    pauses: Some((&self.pause, &self.interrupt)),
7518                    run: &run_id,
7519                    node: "review",
7520                    prompts: &prompts,
7521                    cache: recon_cache.as_deref(),
7522                    round: Some(round),
7523                };
7524                let recon_results = ask_json_wave::<ReviewRevote>(
7525                    jobs,
7526                    Arc::clone(&self.sem),
7527                    review_retries,
7528                    &self.roles.reviewer_roster,
7529                    &recon_ctx,
7530                    &mut recon_quota_losses,
7531                    &mut self.state,
7532                    &|_: &ReviewRevote| Ok(()),
7533                )
7534                .await;
7535                if self.state.resource_block.is_some() {
7536                    return self.stop_for_resource();
7537                }
7538                self.state.quota.extend(recon_quota_losses);
7539
7540                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7541                    let agent_id = seat.agent.clone();
7542                    self.state.seats.insert(seat.key.clone(), seat);
7543                    let mut rec = ReviewRevoteRecord {
7544                        reviewer: r + 1,
7545                        agent: agent_id,
7546                        vote: None,
7547                        reason: String::new(),
7548                        failed: None,
7549                    };
7550                    match res {
7551                        Ok((rv, _)) => {
7552                            rec.vote = Some(rv.vote);
7553                            rec.reason =
7554                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7555                            self.state.event(
7556                                "review",
7557                                format!(
7558                                    "round {round}: reviewer {} revoted {}",
7559                                    r + 1,
7560                                    rv.vote.label()
7561                                ),
7562                            );
7563                        }
7564                        Err(e) => {
7565                            rec.failed = Some(e.to_string());
7566                            self.state.event(
7567                                "review",
7568                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7569                            );
7570                        }
7571                    }
7572                    reconsideration.push(rec);
7573                }
7574            } else if initial_votes.len() > 1 {
7575                self.state.event(
7576                    "review",
7577                    format!(
7578                        "round {round}: votes agreed ({}) — no reconsideration",
7579                        initial_votes[0].label()
7580                    ),
7581                );
7582            }
7583
7584            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7585            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7586            // A round that already has a blocking finding and a round left to
7587            // try is going back to the fixer no matter what `verify.e2e`
7588            // says, so running it first only spends the loop's slowest step
7589            // (minutes, for a Rust repo's full test suite) on a head about
7590            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7591            // runs once a round has no blocking findings left (see
7592            // `round_is_clean`, which a deferred — empty — `e2e` can never
7593            // satisfy since `blocking` is nonzero whenever this branch is
7594            // taken), and `stop_reviewing` forces a real run before it will
7595            // ever read a deferred round as green.
7596            let defer_e2e =
7597                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7598            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7599                let reason =
7600                    format!("{blocking} blocking finding(s) already required a fix this round");
7601                self.state.event(
7602                    "verify",
7603                    format!(
7604                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7605                         {}); it will run once a round has none left",
7606                        short(&head)
7607                    ),
7608                );
7609                (Vec::new(), false, true, Some(reason))
7610            } else {
7611                let e2e_commands = self.state.config.verify.e2e.clone();
7612                let cache_dir = self.state.config.cache_dir();
7613                let context = format!("round {round}");
7614                let wait = self.repository_wait();
7615                let (e2e, verify_retried) = with_cache_lease(
7616                    &mut self.state,
7617                    cache_dir.as_deref(),
7618                    "e2e",
7619                    "e2e",
7620                    &winner.worktree,
7621                    &head,
7622                    verify_timeout,
7623                    &context,
7624                    &wait,
7625                    |state, budget| {
7626                        let shell = shell.clone();
7627                        let e2e_commands = e2e_commands.clone();
7628                        let worktree = winner.worktree.clone();
7629                        let context = context.clone();
7630                        async move {
7631                            run_e2e_with_retry(
7632                                state,
7633                                &shell,
7634                                &e2e_commands,
7635                                &worktree,
7636                                budget,
7637                                &context,
7638                            )
7639                            .await
7640                        }
7641                    },
7642                )
7643                .await?;
7644                (e2e, verify_retried, false, None)
7645            };
7646
7647            let expected = records.len();
7648            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7649            let incomplete = answered < expected;
7650            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7651            let policy = self.state.config.graph.incomplete_review;
7652            let clean = round_is_clean(
7653                blocking,
7654                e2e_ok,
7655                answered,
7656                expected,
7657                round_quota_missing,
7658                policy,
7659            );
7660
7661            let mut round_record = ReviewRound {
7662                round,
7663                head: head.clone(),
7664                verified_head: None,
7665                verified_at: None,
7666                reviews: records,
7667                e2e,
7668                verify_retried,
7669                e2e_deferred,
7670                e2e_defer_reason,
7671                fix: None,
7672                blocking,
7673                answered,
7674                expected,
7675                clean,
7676                progressed: false,
7677                vote_split,
7678                reconsideration,
7679                verdict: None,
7680            };
7681            // The final vote per seat is its revote where reconsideration
7682            // ran and answered, its initial vote otherwise — the same
7683            // fallback `tally` uses for a judge whose private vote failed.
7684            round_record.verdict = ReviewVote::worst(
7685                round_record
7686                    .final_votes()
7687                    .into_iter()
7688                    .map(|(_, _, vote)| vote),
7689            );
7690            // Which commit and when magi actually attempted to check —
7691            // known the moment a command was dispatched against `head`,
7692            // whether or not it finished: a resource-blocked attempt still
7693            // targeted a specific commit at a specific time, and leaving
7694            // that unrecorded is exactly what made `verification_summary`
7695            // report a fresh attempt as "commit unknown ... recorded before
7696            // this was tracked", indistinguishable from a genuinely old,
7697            // untracked record. Only a deferred or unconfigured round never
7698            // ran at all and has nothing to record — see
7699            // `ReviewRound::verified_head`'s own doc.
7700            if !matches!(
7701                round_record.e2e_status(),
7702                E2eStatus::Deferred | E2eStatus::NotConfigured
7703            ) {
7704                round_record.verified_head = Some(head.clone());
7705                round_record.verified_at = Some(Timestamp::now());
7706            }
7707
7708            if incomplete {
7709                let missing: Vec<String> = round_record
7710                    .reviews
7711                    .iter()
7712                    .filter(|r| r.failed.is_some())
7713                    .map(|r| format!("review-{}", r.reviewer))
7714                    .collect();
7715                self.state.event(
7716                    "review",
7717                    format!(
7718                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7719                        missing.join(", ")
7720                    ),
7721                );
7722            }
7723
7724            if clean {
7725                self.state.event(
7726                    "review",
7727                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7728                        format!(
7729                            "round {round}: clean (warn policy, incomplete panel) — no \
7730                             blocking findings from the seats that answered, verification green"
7731                        )
7732                    } else if incomplete {
7733                        format!(
7734                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7735                             quorum) — no blocking findings from the seats that answered, \
7736                             verification green",
7737                            expected - answered
7738                        )
7739                    } else {
7740                        format!("round {round}: clean — no blocking findings, verification green")
7741                    },
7742                );
7743                self.state.reviews.push(round_record);
7744                self.state.status = RunStatus::Gating;
7745                self.state.save()?;
7746                return Ok(());
7747            }
7748
7749            // Nothing was raised and verification passed, but not every seat
7750            // answered and `round_is_clean` still refused to call it clean —
7751            // either a seat is missing for a reason other than its own quota
7752            // (a crash, a timeout, unparsable output — worth another try), or
7753            // every seat that could have answered lost its quota and nobody
7754            // is left to decide on: re-review rather than send the fixer
7755            // after a round with nothing to fix.
7756            if incomplete && blocking == 0 && e2e_ok {
7757                self.state.reviews.push(round_record);
7758                self.state.save()?;
7759                if round == max_rounds {
7760                    self.state.status = RunStatus::Blocked;
7761                    self.state.event(
7762                        "review",
7763                        format!(
7764                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7765                             refusing to call it clean",
7766                            expected - answered
7767                        ),
7768                    );
7769                    return Ok(());
7770                }
7771                continue;
7772            }
7773
7774            // Nothing for the fixer to act on (`blocking == 0`) and the only
7775            // reason this round is not clean is that magi itself never got
7776            // a command to run — the shared build cache, not the patch (see
7777            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7778            // the fixer would invite a change to appease contention that has
7779            // nothing to do with the diff, and would leave this attempt
7780            // sitting in the next round's prompt as if it were about an
7781            // earlier, superseded commit rather than what it actually is:
7782            // the same head, still waiting to be checked. Wait for it the
7783            // same way the final round's own contention is already handled,
7784            // whatever round this happens to be.
7785            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7786                self.state.reviews.push(round_record);
7787                return self
7788                    .stop_unresolved_review(
7789                        "the round's own verification could not run",
7790                        &shell,
7791                        &winner.worktree,
7792                    )
7793                    .await;
7794            }
7795
7796            if round == max_rounds {
7797                self.state.reviews.push(round_record);
7798                return self
7799                    .stop_reviewing(
7800                        &format!(
7801                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7802                        ),
7803                        &shell,
7804                        &winner.worktree,
7805                    )
7806                    .await;
7807            }
7808
7809            if !self.fix_review_round(&winner, round_record).await? {
7810                return Ok(());
7811            }
7812        }
7813        Ok(())
7814    }
7815
7816    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7817    /// False means admission or continuation policy stopped the review loop.
7818    async fn fix_review_round(
7819        &mut self,
7820        winner: &Candidate,
7821        mut round_record: ReviewRound,
7822    ) -> Result<bool> {
7823        let round = round_record.round;
7824        let shell = self.state.config.shell();
7825        let step = self.run_fix_round(winner, &mut round_record).await?;
7826        if matches!(step, FixStep::Resource) {
7827            self.stop_for_resource()?;
7828            return Ok(false);
7829        }
7830        if self.state.reviews.len() < round {
7831            self.state.reviews.push(round_record);
7832        } else {
7833            self.state.reviews[round - 1] = round_record;
7834        }
7835        self.state.save()?;
7836
7837        // The fixer's own report never came back this round, even after
7838        // `continue_fix_report`'s own budget was spent on it — not an
7839        // ordinary "no report" (dropped stream, quota, plain failure),
7840        // which already reads that way and is left to the existing round
7841        // budget. Stopping here, rather than opening another round, is
7842        // what keeps a next reviewer/fixer wave from ever being
7843        // dispatched onto `winner.worktree` while whatever the seat's
7844        // last call may still have running there is unaccounted for: no
7845        // process liveness check exists (and none is being added — see
7846        // AGENTS.md/this task's own scope), so the only way to honour
7847        // "nothing starts before a valid report returns" is to not start
7848        // anything further on this worktree from this run at all.
7849        if matches!(step, FixStep::Recorded { report_lost: true }) {
7850            self.stop_reviewing(
7851                "the fixer's adoption report never came back, even after resuming its \
7852                         own seat; refusing to start another round against the same worktree \
7853                         while that is unresolved",
7854                &shell,
7855                &winner.worktree,
7856            )
7857            .await?;
7858            return Ok(false);
7859        }
7860
7861        let streak = self
7862            .state
7863            .reviews
7864            .iter()
7865            .rev()
7866            .take_while(|r| !r.progressed)
7867            .count();
7868        if streak >= STAGNANT_LIMIT {
7869            self.stop_reviewing(
7870                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7871                &shell,
7872                &winner.worktree,
7873            )
7874            .await?;
7875            return Ok(false);
7876        }
7877        Ok(true)
7878    }
7879
7880    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7881    async fn run_fix_round(
7882        &mut self,
7883        winner: &Candidate,
7884        round_record: &mut ReviewRound,
7885    ) -> Result<FixStep> {
7886        let round = round_record.round;
7887        let max_rounds = self.state.config.graph.review_rounds;
7888        let run_id = self.state.id.clone();
7889        let prompts = self.state.config.prompts.clone();
7890        let language = self.state.config.graph.language.clone();
7891        let sessions = self.state.config.graph.sessions;
7892        let artifacts = self.merge_repair_artifacts();
7893        let base = self.landing_base();
7894        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7895        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7896        let this_round_verification = round_record.verification_summary(&head);
7897        let all_findings: Vec<_> = round_record
7898            .reviews
7899            .iter()
7900            .flat_map(|r| r.findings.iter().cloned())
7901            .collect();
7902        // Fix. The winner's own implementer seat continues its conversation:
7903        // the competition is over, so context is pure benefit now.
7904        let blocking_findings: Vec<_> = all_findings
7905            .iter()
7906            .filter(|f| f.severity.blocks())
7907            .cloned()
7908            .collect();
7909        let fix_prompt = prompt::fix_in(
7910            &self.state.instruction,
7911            &blocking_findings,
7912            this_round_verification.as_ref(),
7913            round,
7914            max_rounds,
7915            &language,
7916            &self.state.config.graph.github_language,
7917        );
7918        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7919        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7920        let (job, seat, out) = self
7921            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7922                prompt: fix_prompt.clone(),
7923                spec,
7924                seat,
7925                cwd: winner.worktree.clone(),
7926                timeout,
7927                allow_write: true,
7928                sessions,
7929                artifacts: artifacts.clone(),
7930                stem: format!("fix-{round}"),
7931                handover: None,
7932            })
7933            .await;
7934        // Not asked: the round is not recorded (so a resume repeats it),
7935        // no fixer was handed over to, and the seat is left as it was.
7936        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7937            return Ok(FixStep::Resource);
7938        }
7939        let agent_id = seat.agent.clone();
7940
7941        let mut fix = FixRecord {
7942            agent: agent_id,
7943            addressed: Vec::new(),
7944            rejected: Vec::new(),
7945            notes: String::new(),
7946            committed: false,
7947            failed: None,
7948            duration_ms: 0,
7949            continuation: None,
7950        };
7951        let mut continuation = ContinuationRecord::not_needed();
7952        let mut final_seat = seat.clone();
7953        match out {
7954            AgentOutcome::Ok(o) => {
7955                fix.duration_ms = o.duration_ms;
7956                let parsed = verdict::extract_json::<FixReport>(&o.text);
7957                // A parsed report standing next to a command this same
7958                // reply's own CLI never confirmed the exit status of is
7959                // not a resolved answer — the identical `CommandEvidence`
7960                // `state.jobs` renders, read here instead of only on
7961                // display, per the completion judgment and the shown
7962                // record needing to agree.
7963                let incomplete_reason = match &parsed {
7964                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7965                        "the reply parsed, but it reported a command whose own CLI never \
7966                             confirmed an exit status"
7967                            .to_owned(),
7968                    ),
7969                    Ok(_) => None,
7970                    Err(e) => Some(e.to_string()),
7971                };
7972                match incomplete_reason {
7973                    None => {
7974                        let report = parsed.expect("checked Ok above");
7975                        fix.addressed = report.addressed;
7976                        fix.rejected = report.rejected;
7977                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7978                    }
7979                    Some(reason) => {
7980                        let (resumed_seat, resolved, failure, cont) = self
7981                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7982                            .await;
7983                        fix.duration_ms += cont.cumulative_wait_ms;
7984                        continuation = cont;
7985                        final_seat = resumed_seat;
7986                        match resolved {
7987                            Some(report) => {
7988                                fix.addressed = report.addressed;
7989                                fix.rejected = report.rejected;
7990                                fix.notes =
7991                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7992                            }
7993                            None => fix.failed = failure,
7994                        }
7995                    }
7996                }
7997            }
7998            // The CLI's raw error JSON is not a fix report to parse.
7999            AgentOutcome::Dropped(o) => {
8000                fix.duration_ms = o.duration_ms;
8001                let why = o
8002                    .dropped
8003                    .as_ref()
8004                    .map(|d| d.why.as_str())
8005                    .unwrap_or("the CLI ended the stream without delivering its answer");
8006                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8007            }
8008            AgentOutcome::Quota(o) => {
8009                self.state.quota.push(QuotaLoss::from_report(
8010                    final_seat.key.clone(),
8011                    "fix",
8012                    o.quota.as_ref(),
8013                ));
8014                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8015            }
8016            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8017        }
8018        if self.state.resource_block.is_some() {
8019            self.state.seats.insert(final_seat.key.clone(), final_seat);
8020            return Ok(FixStep::Resource);
8021        }
8022        fix.continuation = Some(continuation);
8023        self.state.seats.insert(final_seat.key.clone(), final_seat);
8024        let coordination = self.coordinate(&winner.worktree).await?;
8025        if let Ok(r) = git::rescue_commit(
8026            &winner.worktree,
8027            &format!("magi: review round {round} fixes (uncommitted work)"),
8028        )
8029        .await
8030        {
8031            self.state.note_withheld("fix", &r.withheld);
8032        }
8033        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8034        fix.committed = after != before;
8035        // Judged by what `git` says moved against base, never by the
8036        // fixer's own `addressed`/`rejected` count — see
8037        // `ReviewRound::progressed`. Propagated with `?`, the same as the
8038        // `patch` snapshot above: swallowing this error would default
8039        // `diff_after` to empty, which almost always differs from a
8040        // non-empty `patch` and reads as "progressed" — exactly backwards
8041        // for a `git` failure the stagnation check cannot see through.
8042        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8043        drop(coordination);
8044        let progressed = diff_after != patch;
8045        let commit_note = if fix.committed {
8046            "committed"
8047        } else {
8048            "NO new commit"
8049        };
8050        let tree_note = if progressed {
8051            "changed vs base"
8052        } else {
8053            "unchanged vs base"
8054        };
8055        self.state.event(
8056            "fix",
8057            match &fix.failed {
8058                // Distinct on purpose from "0 addressed, 0 rejected": the
8059                // fixer's own diff still landed (blocking counts do keep
8060                // falling round over round), only its adoption report did
8061                // not come back, so this must never read like every
8062                // finding was reviewed and declined.
8063                Some(reason) => {
8064                    format!(
8065                        "round {round}: fixer's adoption report was lost ({reason}); \
8066                             {commit_note}, tree {tree_note}"
8067                    )
8068                }
8069                None => format!(
8070                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8071                         {tree_note}{}",
8072                    fix.addressed.len(),
8073                    fix.rejected.len(),
8074                    if continuation.outcome == ContinuationOutcome::Resumed {
8075                        format!(
8076                            " (adoption report recovered after {} continuation(s))",
8077                            continuation.attempts
8078                        )
8079                    } else {
8080                        String::new()
8081                    },
8082                ),
8083            },
8084        );
8085        round_record.fix = Some(fix);
8086        round_record.progressed = progressed;
8087        Ok(FixStep::Recorded {
8088            report_lost: matches!(
8089                continuation.outcome,
8090                ContinuationOutcome::Exhausted
8091                    | ContinuationOutcome::QuotaLost
8092                    | ContinuationOutcome::NoSession
8093            ),
8094        })
8095    }
8096
8097    /// Decide, from the last recorded round's own verification, whether
8098    /// stopping the review loop is a hand-off or a genuine block.
8099    ///
8100    /// Called once the loop has given up trying — the round budget is spent,
8101    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8102    /// findings still open, never while a round is still clean or the
8103    /// incomplete-panel case handled inline above. Gate and e2e are facts
8104    /// about the tree; a lingering review finding is an opinion, and this
8105    /// workload's own `magi stats` puts reviewer precision low enough
8106    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8107    /// must not by itself stand between a green, verified change and the
8108    /// human who decides what to do with it. A red e2e is not an opinion, so
8109    /// that case still blocks, with the failing command and a tail of its
8110    /// output recorded here rather than left in `run.json` for someone to go
8111    /// find.
8112    ///
8113    /// A round that deferred its own e2e (see [`Config::graph`]'s
8114    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8115    /// only because nothing ran, and treating an empty list as a passing one
8116    /// here is exactly the "deferred painted green" bug this function exists
8117    /// to not have. When the last round's own verification never resolved —
8118    /// deferred on purpose, or a real attempt the shared build cache blocked
8119    /// — this makes (or retries) the real run, on the actual worktree this
8120    /// loop is about to stop touching, before deciding anything. A
8121    /// resource-blocked attempt is likewise never read as either green or
8122    /// red: it is evidence about the machine, not the patch (see
8123    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8124    /// blocked cache leaves this call without deciding rather than guessing
8125    /// — the caller retries on a later reentry.
8126    /// Record, once, that the review loop handed off over a blocking finding
8127    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8128    /// `land` asks the owner even with `land_approval` off. Called from every
8129    /// path that concludes `Gating`; a reentry keeps the first record.
8130    fn record_contested_handoff(&mut self) {
8131        if self.state.contested_handoff.is_some() {
8132            return;
8133        }
8134        let Some(contested) = self
8135            .state
8136            .reviews
8137            .last()
8138            .and_then(ReviewRound::contested_handoff)
8139        else {
8140            return;
8141        };
8142        self.state.event(
8143            "review",
8144            format!(
8145                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8146                 wait for the owner's approval",
8147                contested.findings.len(),
8148                contested.rejecters.len()
8149            ),
8150        );
8151        self.state.contested_handoff = Some(contested);
8152    }
8153
8154    async fn stop_unresolved_review(
8155        &mut self,
8156        why: &str,
8157        shell: &[String],
8158        worktree: &Path,
8159    ) -> Result<()> {
8160        self.stop_reviewing_inner(why, shell, worktree, true).await
8161    }
8162
8163    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8164        self.stop_reviewing_inner(why, shell, worktree, false).await
8165    }
8166
8167    async fn stop_reviewing_inner(
8168        &mut self,
8169        why: &str,
8170        shell: &[String],
8171        worktree: &Path,
8172        repairable: bool,
8173    ) -> Result<()> {
8174        let round_idx = self.state.reviews.len() - 1;
8175        // A deferred round and a resource-blocked one are the same shape
8176        // here: neither has a real result yet, and both get one more
8177        // attempt. Read off `e2e_status` — the single source for this —
8178        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8179        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8180        // still retries instead of being read as a settled result the
8181        // instant it stops being empty.
8182        let needs_catchup_run = matches!(
8183            self.state.reviews[round_idx].e2e_status(),
8184            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8185        );
8186        if needs_catchup_run {
8187            let round = self.state.reviews[round_idx].round;
8188            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8189            let commands = self.state.config.verify.e2e.clone();
8190            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8191            let cache_dir = self.state.config.cache_dir();
8192            let context = format!(
8193                "round {round}: verification unresolved, catching up before the final decision"
8194            );
8195            let wait = self.repository_wait();
8196            let (outcomes, verify_retried) = with_cache_lease(
8197                &mut self.state,
8198                cache_dir.as_deref(),
8199                "e2e",
8200                "e2e",
8201                worktree,
8202                &attempted_head,
8203                timeout,
8204                &context,
8205                &wait,
8206                |state, budget| {
8207                    let shell = shell.to_vec();
8208                    let commands = commands.clone();
8209                    let context = context.clone();
8210                    async move {
8211                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8212                            .await
8213                    }
8214                },
8215            )
8216            .await?;
8217            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8218                self.state.record_base_progress(&attempted_head);
8219            }
8220            let last = &mut self.state.reviews[round_idx];
8221            last.e2e = outcomes;
8222            last.verify_retried = verify_retried;
8223            // Always the commit and time this attempt actually targeted,
8224            // whether or not it happens to equal the reviewed `head` and
8225            // whether or not a command finished — see
8226            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8227            // attempt is recorded too, so a later reader sees "attempted
8228            // again at T2" rather than silence.
8229            last.verified_head = Some(attempted_head);
8230            last.verified_at = Some(Timestamp::now());
8231            if verify_inconclusive(&last.e2e) {
8232                // Still not a real result: `e2e_deferred` is left exactly
8233                // as it was, so `needs_catchup_run` above reads
8234                // `ResourceBlocked` (via `e2e_status`, which checks
8235                // `resource_blocked` before `e2e_deferred`) and retries
8236                // again on the next reentry, rather than recording
8237                // contention as a red e2e and blocking the run on it.
8238                self.state.save()?;
8239                return Ok(());
8240            }
8241            last.e2e_deferred = false;
8242            // A complete panel may have been saved as unclean solely because
8243            // verification could not run. Refresh that verdict with the new
8244            // outcomes before a granted repair consumes the round's result.
8245            // Incomplete panels retain their original policy/quota decision.
8246            if last.answered == last.expected {
8247                last.clean = round_is_clean(
8248                    last.blocking,
8249                    last.e2e.iter().all(CommandOutcome::ok),
8250                    last.answered,
8251                    last.expected,
8252                    0,
8253                    self.state.config.graph.incomplete_review,
8254                );
8255            }
8256        }
8257        let last = &self.state.reviews[round_idx];
8258        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8259
8260        match last.e2e_status() {
8261            E2eStatus::Failed => {
8262                let red: Vec<String> = last
8263                    .e2e
8264                    .iter()
8265                    .filter(|o| !o.ok())
8266                    .map(|o| {
8267                        format!(
8268                            "`{}` -> {:?}\n{}",
8269                            o.command,
8270                            o.code,
8271                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8272                        )
8273                    })
8274                    .collect();
8275                let can_repair = last.blocking == 0 && last.fix.is_none();
8276                self.state.reviews[round_idx].clean = false;
8277                self.state
8278                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8279                self.state.status = RunStatus::Blocked;
8280                if repairable
8281                    && can_repair
8282                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8283                {
8284                    self.state.status = RunStatus::Reviewing;
8285                }
8286            }
8287            // `needs_catchup_run` above already retried once this call; if
8288            // it is still blocked, this is magi's own admission it could
8289            // not get a command to run, never a verdict on the patch — the
8290            // run is left exactly where a later reentry can retry again.
8291            E2eStatus::ResourceBlocked => {
8292                self.state.event(
8293                    "review",
8294                    format!(
8295                        "{why}; e2e could not run (resource unavailable); not \
8296                         deciding yet"
8297                    ),
8298                );
8299            }
8300            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8301                self.state.event(
8302                    "review",
8303                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8304                );
8305                self.record_contested_handoff();
8306                self.state.status = RunStatus::Gating;
8307            }
8308        }
8309        self.state.save()?;
8310        Ok(())
8311    }
8312
8313    // ----------------------------------------------------------------- gate
8314
8315    async fn gate(&mut self) -> Result<()> {
8316        if self.hold_base_sync().await? {
8317            return Ok(());
8318        }
8319        // Judged by the review record itself, not by `status`: a solo
8320        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8321        // reentry (see `judge`), and trusting it here is exactly how a run
8322        // that exhausted its review budget got gated and merged a second
8323        // time around. `review_conclusion` recomputes the review loop's own
8324        // verdict from the round records themselves — `Gating` for a clean
8325        // round or a hand-off (see `stop_reviewing`), anything else means the
8326        // loop is still going or genuinely blocked.
8327        // A base the winner could not be replayed onto is a decision, not a
8328        // round: there is no landing tree to gate. Read as its own record for
8329        // the same reason the review verdict is.
8330        if self.state.status == RunStatus::Failed
8331            || self
8332                .state
8333                .base_sync
8334                .as_ref()
8335                .is_some_and(|s| s.conflict.is_some())
8336            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8337                != Some(RunStatus::Gating)
8338        {
8339            return Ok(());
8340        }
8341        let Some(winner) = self.state.winner().cloned() else {
8342            return Ok(());
8343        };
8344        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8345        self.state.reconcile_verification_head(&head);
8346        self.state.save()?;
8347        let wait = self.repository_wait();
8348        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8349            return Ok(());
8350        }
8351        if self.state.gate_ran {
8352            // `review_loop` derives its conclusion from the clean review
8353            // record on every reentry and therefore puts a completed run back
8354            // in `Gating`. A recorded gate is a stronger, terminal fact:
8355            // retain its original command output (or lack of any, for a repo
8356            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8357            // doc) and restore `Blocked` on a real failure rather than
8358            // pretending the command is still running or running it a second
8359            // time. `gate_ran == false` remains the only shape — unattempted,
8360            // or a resource-blocked retry — that may still need to execute a
8361            // command.
8362            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8363                self.state.status = RunStatus::Blocked;
8364                self.state.save()?;
8365            }
8366            return Ok(());
8367        }
8368        self.state.status = RunStatus::Gating;
8369        let mut outcomes = self.run_gate(&winner).await?;
8370        let mut attempted_head = self.state.gate_head.clone();
8371        loop {
8372            // A resource-blocked outcome means the gate command never actually
8373            // ran - the shared build cache could not be acquired or confirmed
8374            // fresh in time - which is evidence about the machine, not about
8375            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8376            // Recording it as a red gate would mark a run `Blocked` on nothing
8377            // but contention magi has already logged; leaving `self.state.gate`
8378            // empty and `self.state.gate_ran` false instead keeps the shape
8379            // this function already treats as "still needs to run" (see the
8380            // early-return above), so the next call retries the command
8381            // rather than concluding anything.
8382            if verify_inconclusive(&outcomes) {
8383                self.state.save()?;
8384                return Ok(());
8385            }
8386            if outcomes.iter().all(CommandOutcome::ok) {
8387                break;
8388            }
8389            let fixes_before = self.state.gate_fixes.len();
8390            match self.gate_fix_round(&winner, &outcomes).await? {
8391                GateFix::Retry => {
8392                    outcomes = self.run_gate(&winner).await?;
8393                    attempted_head = self.state.gate_head.clone();
8394                }
8395                GateFix::Stop => {
8396                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8397                    if self.state.gate_fixes.len() > fixes_before
8398                        && attempted_head.as_deref() != Some(current.as_str())
8399                        && verify_current_head(&mut self.state, &winner, &wait).await?
8400                    {
8401                        // A new SHA owes a gate even when a spent fixer changed only
8402                        // commit metadata. This does not grant another repair round.
8403                        outcomes = self.run_gate(&winner).await?;
8404                        attempted_head = self.state.gate_head.clone();
8405                    } else {
8406                        break;
8407                    }
8408                }
8409                GateFix::Defer => {
8410                    self.state.save()?;
8411                    return Ok(());
8412                }
8413            }
8414        }
8415        let passed = outcomes.iter().all(CommandOutcome::ok);
8416        self.state.gate = outcomes;
8417        self.state.gate_ran = true;
8418        self.state.gate_head = attempted_head;
8419        if !self.state.gate.is_empty()
8420            && !verify_inconclusive(&self.state.gate)
8421            && let Some(head) = self.state.gate_head.clone()
8422        {
8423            self.state.record_base_progress(&head);
8424        }
8425        self.state.save()?;
8426        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8427            return Ok(());
8428        }
8429        if !passed {
8430            self.state.status = RunStatus::Blocked;
8431            let spent = self.state.gate_fixes.len();
8432            self.state.event(
8433                "gate",
8434                if spent == 0 {
8435                    "gate failed; not merging".to_owned()
8436                } else {
8437                    format!("gate failed after {spent} gate-fix round(s); not merging")
8438                },
8439            );
8440        }
8441        self.state.save()?;
8442        Ok(())
8443    }
8444
8445    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8446    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8447    /// after review is clean and never on a candidate awaiting judging.
8448    ///
8449    /// Never fails the run: a non-zero exit or timeout is a warning and a
8450    /// recorded outcome, and the gate remains the single arbiter. Nothing
8451    /// configured means nothing happens - no event, no commit. `commit_all`
8452    /// commits any leftover change under the neutral identity and returns
8453    /// `false` when the tree is clean, so no empty commit is ever made.
8454    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8455        let commands = state.config.verify.pre_gate.clone();
8456        if commands.is_empty() {
8457            return;
8458        }
8459        let shell = state.config.shell();
8460        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8461        let (outcomes, _) = run_commands(
8462            state,
8463            "pre_gate",
8464            "pre_gate",
8465            0,
8466            &shell,
8467            &commands,
8468            &winner.worktree,
8469            timeout,
8470        )
8471        .await;
8472        for o in &outcomes {
8473            if !o.ok() {
8474                tracing::warn!(
8475                    "pre_gate `{}` failed ({:?}); the gate decides",
8476                    o.command,
8477                    o.code
8478                );
8479            }
8480            state.event(
8481                "pre_gate",
8482                format!(
8483                    "`{}` -> {}",
8484                    o.command,
8485                    if o.ok() {
8486                        "pass".to_owned()
8487                    } else {
8488                        format!(
8489                            "FAIL ({:?})\n{}",
8490                            o.code,
8491                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8492                        )
8493                    }
8494                ),
8495            );
8496        }
8497        state.pre_gate = outcomes;
8498        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8499            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8500                Ok(head) => {
8501                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8502                    state.pre_gate_commit = Some(head);
8503                }
8504                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8505            },
8506            Ok(false) => {}
8507            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8508        }
8509        if let Err(e) = state.save() {
8510            tracing::warn!("could not persist the pre_gate record: {e:#}");
8511        }
8512    }
8513
8514    /// Run `verify.gate` once against the winner's current tree, logging one
8515    /// event per command. Empty when nothing is configured.
8516    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8517        let wait = self.repository_wait();
8518        run_landing_gate(&mut self.state, winner, &wait).await
8519    }
8520
8521    /// One bounded fix round for a failing gate.
8522    ///
8523    /// The fixer is told the failure came from the gate itself, not from a
8524    /// reviewer, and is shown the failed commands, their exit codes and a tail
8525    /// of their output - whatever `[verify].gate` holds, nothing here knows
8526    /// what those commands run. Only a normal non-zero exit that printed
8527    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8528    /// command or a full disk says nothing about the code, and a fixer sent
8529    /// after it can only appease the machine. The round is judged by what git
8530    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8531    /// again before the gate does, so a fix cannot trade a green gate for a
8532    /// red e2e unnoticed.
8533    async fn gate_fix_round(
8534        &mut self,
8535        winner: &Candidate,
8536        outcomes: &[CommandOutcome],
8537    ) -> Result<GateFix> {
8538        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8539            g.phase == crate::repair_budget::Phase::Fixing
8540                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8541        }) {
8542            self.state.gate_fixes.len().saturating_add(1)
8543        } else {
8544            self.state.config.graph.gate_fix_rounds
8545        };
8546        let spent = self.state.gate_fixes.len();
8547        if spent >= cap {
8548            if cap > 0 {
8549                self.state.event(
8550                    "gate",
8551                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8552                );
8553            }
8554            return Ok(GateFix::Stop);
8555        }
8556        if !gate_fixable(outcomes) {
8557            self.state.event(
8558                "gate",
8559                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8560                 command or similar); not spending a fix round on it",
8561            );
8562            return Ok(GateFix::Stop);
8563        }
8564        let min_free = self.state.config.disk.min_free_bytes;
8565        if min_free > 0 {
8566            match crate::disk::free_bytes(&winner.worktree) {
8567                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8568                Ok(free) => {
8569                    self.state.event(
8570                        "gate",
8571                        format!(
8572                            "only {free} bytes free ({min_free} required by `[disk] \
8573                             min_free_bytes`); not spending a fix round on a failure the disk \
8574                             may explain"
8575                        ),
8576                    );
8577                    return Ok(GateFix::Stop);
8578                }
8579                Err(e) => {
8580                    self.state.event(
8581                        "gate",
8582                        format!("free disk space could not be measured ({e:#}); no fix round"),
8583                    );
8584                    return Ok(GateFix::Stop);
8585                }
8586            }
8587        }
8588
8589        let attempt = spent + 1;
8590        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8591        let base = self.landing_base();
8592        let fix_prompt = prompt::gate_fix_in(
8593            &self.state.instruction,
8594            &failed,
8595            attempt,
8596            cap,
8597            &self.state.config.graph.language,
8598            &self.state.config.graph.github_language,
8599        );
8600        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8601        let sessions = self.state.config.graph.sessions;
8602        let artifacts = self.merge_repair_artifacts();
8603        self.state.event(
8604            "gate",
8605            format!("gate failed; gate-fix round {attempt} of {cap}"),
8606        );
8607        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8608        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8609        // Persist before invoking the fixer: it may commit and then be
8610        // interrupted before its result (or the following e2e) is saved.
8611        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8612            head: before.clone(),
8613            passed: false,
8614            outcomes: Vec::new(),
8615        });
8616        self.state.save()?;
8617        let (_, seat, out) = self
8618            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8619                prompt: fix_prompt.clone(),
8620                spec,
8621                seat,
8622                cwd: winner.worktree.clone(),
8623                timeout,
8624                allow_write: true,
8625                sessions,
8626                artifacts: artifacts.clone(),
8627                stem: format!("gate-fix-{attempt}"),
8628                handover: None,
8629            })
8630            .await;
8631        // The fixer was never asked: no round is spent (nothing is recorded)
8632        // and a later reentry asks the same agent again.
8633        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8634            return Ok(GateFix::Defer);
8635        }
8636        let mut record = GateFixRecord {
8637            agent: seat.agent.clone(),
8638            failed,
8639            notes: String::new(),
8640            committed: false,
8641            error: None,
8642        };
8643        match out {
8644            AgentOutcome::Ok(o) => {
8645                // A missing report is not a failed fix: the round is judged
8646                // by the tree below, and the report only carries prose.
8647                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8648                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8649                }
8650            }
8651            AgentOutcome::Dropped(_) => {
8652                record.error = Some("the CLI dropped the stream".to_owned());
8653            }
8654            AgentOutcome::Quota(o) => {
8655                self.state.quota.push(QuotaLoss::from_report(
8656                    seat.key.clone(),
8657                    "gate-fix",
8658                    o.quota.as_ref(),
8659                ));
8660                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8661            }
8662            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8663        }
8664        self.state.seats.insert(seat.key.clone(), seat);
8665        let coordination = self.coordinate(&winner.worktree).await?;
8666        if let Ok(r) = git::rescue_commit(
8667            &winner.worktree,
8668            &format!("magi: gate fix {attempt} (uncommitted work)"),
8669        )
8670        .await
8671        {
8672            self.state.note_withheld("gate-fix", &r.withheld);
8673        }
8674        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8675        record.committed = after != before;
8676        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8677        let note = record.error.clone();
8678        self.state.gate_fixes.push(record);
8679        self.state.save()?;
8680        if !changed {
8681            self.state.event(
8682                "gate",
8683                match note {
8684                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8685                    None => format!("gate-fix round {attempt}: the tree did not change"),
8686                },
8687            );
8688            return Ok(GateFix::Stop);
8689        }
8690        self.state.event(
8691            "gate",
8692            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8693        );
8694
8695        drop(coordination);
8696        let wait = self.repository_wait();
8697        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8698    }
8699
8700    // ---------------------------------------------------------------- merge
8701
8702    /// One read-only rewrite by the summary's author; a text the gate still
8703    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8704    /// applies on their word or on silence. `None` means the run is parked on
8705    /// that question and nothing may be pushed or opened yet.
8706    async fn guarded_pr_message(
8707        &mut self,
8708        winner: &Candidate,
8709        facts: Option<&BranchFacts>,
8710        posting: bool,
8711    ) -> Result<Option<PrMessage>> {
8712        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8713        let head = match final_head.as_deref() {
8714            Some(h) => crate::pr_outcome::FinalHead::At(h),
8715            None => crate::pr_outcome::FinalHead::Unknown,
8716        };
8717        // Only a pull request that is about to be posted pays for a summary.
8718        let summarized = if posting {
8719            Some(self.summarize_change(winner, final_head.as_deref()).await)
8720        } else {
8721            None
8722        };
8723        // A busy shared resource is not a permanent cause: publishing an
8724        // "unavailable" description would stick (a later head move never
8725        // edits the body). Stop before anything is pushed or opened; the
8726        // run is `Blocked` and a resume asks the summary again.
8727        if self.state.resource_block.is_some() {
8728            self.state.event(
8729                "pr-summary",
8730                "a shared resource was busy; nothing was published, resume to summarize again",
8731            );
8732            return Ok(None);
8733        }
8734        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8735            anyhow::bail!(
8736                "the final commit moved during public description generation; retry before publishing"
8737            );
8738        }
8739        let note = match (&summarized, &self.state.change_summary) {
8740            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8741            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8742            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8743            (None, _) => ChangeNote::NotAsked,
8744        };
8745        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8746        if !posting {
8747            let identity = crate::scrub::Identity::current();
8748            return Ok(Some(PrMessage {
8749                title: crate::scrub::scrub(&pr.title, &identity),
8750                body: crate::scrub::scrub(&pr.body, &identity),
8751            }));
8752        }
8753        // This very text was already rejected and asked about: no second
8754        // judge call, rewrite or question, just carry the record on.
8755        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8756        if let Some(known) = self
8757            .state
8758            .github_text
8759            .clone()
8760            .filter(|g| g.fingerprint == raw_fp)
8761        {
8762            return self.settle_withheld(winner, &pr, known).await;
8763        }
8764        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8765        let mut violations =
8766            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8767        let identity = crate::scrub::Identity::current();
8768        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8769            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8770            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8771        {
8772            violations.push(crate::github_text::Violation::SensitiveData);
8773        }
8774        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8775        // only a language violation starts one; sensitive data alone goes
8776        // straight to the owner, with the redacted text as the fallback.
8777        let has_language = violations
8778            .iter()
8779            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8780        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8781            self.state.event(
8782                "github-text",
8783                if has_language {
8784                    format!("description rejected: {violations:?}; requesting one rewrite")
8785                } else {
8786                    format!("description rejected: {violations:?}; asking the owner")
8787                },
8788            );
8789            let mut rewritten = false;
8790            let mut rejected_rewrite = None;
8791            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8792            {
8793                let output = text;
8794                if let Some(title) = summary_title(&output) {
8795                    // Judge the redacted candidate that will actually be posted.
8796                    // Sensitive data alone must not discard a useful rewrite.
8797                    let raw_body = summary_without_title(&output);
8798                    let clean_title = crate::scrub::scrub(&title, &identity);
8799                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8800                    let redacted = clean_title != title || body != raw_body;
8801                    let title = clean_title;
8802                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8803                    if !complete {
8804                        self.state.event(
8805                            "github-text",
8806                            "description rewrite omitted required findings",
8807                        );
8808                    }
8809                    if !body.trim().is_empty() {
8810                        body.push_str("\n\n");
8811                        body.push_str(&public_pr_evidence(&self.state, head));
8812                    }
8813                    let marker = format!("magi:run/{}", self.state.id);
8814                    if !body.trim().is_empty() && !body.contains(&marker) {
8815                        body.push_str(&format!("\n\n{marker}"));
8816                    }
8817                    let rewrite = PrMessage {
8818                        title: title.clone(),
8819                        body: body.clone(),
8820                    };
8821                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8822                    let mut failed = crate::github_text::check_for(
8823                        &self.state.config.graph,
8824                        &title,
8825                        &body,
8826                        redecision,
8827                    );
8828                    if (!crate::github_text::shareable(&title)
8829                        || !crate::github_text::shareable(&body))
8830                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8831                    {
8832                        failed.push(crate::github_text::Violation::SensitiveData);
8833                    }
8834                    if complete
8835                        && !title.trim().is_empty()
8836                        && !body.trim().is_empty()
8837                        && !failed.is_empty()
8838                    {
8839                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8840                            title: title.clone(),
8841                            body: body.clone(),
8842                            failed: failed
8843                                .iter()
8844                                .map(|v| crate::github_text::category(*v).to_owned())
8845                                .collect(),
8846                        });
8847                    }
8848                    if complete
8849                        && !title.trim().is_empty()
8850                        && !raw_body.trim().is_empty()
8851                        && failed.is_empty()
8852                    {
8853                        decision = redecision;
8854                        pr = PrMessage { title, body };
8855                        rewritten = true;
8856                        self.state
8857                            .event("github-text", "description rewrite passed");
8858                        if redacted {
8859                            self.state.event(
8860                                "github-text",
8861                                "sensitive data removed from description rewrite",
8862                            );
8863                        }
8864                    }
8865                }
8866            }
8867            if !rewritten {
8868                self.state.event(
8869                    "github-text",
8870                    "description rewrite unavailable or rejected; asking the owner",
8871                );
8872                if let Some(w) =
8873                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8874                {
8875                    // A different text than the one asked about before: that
8876                    // question no longer describes anything.
8877                    self.retire_github_text_question("the withheld text changed");
8878                    let record = crate::run::GithubTextAsk {
8879                        rewrite: rejected_rewrite,
8880                        fingerprint: raw_fp,
8881                        title: w.title,
8882                        body: w.body,
8883                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8884                        question: None,
8885                        asks: 0,
8886                        resolved: false,
8887                        chosen_title: None,
8888                        choice: None,
8889                        publication: None,
8890                    };
8891                    return self.settle_withheld(winner, &pr, record).await;
8892                }
8893            }
8894        }
8895        let (title, body) =
8896            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8897        Ok(Some(PrMessage { title, body }))
8898    }
8899
8900    /// Describe the whole change from its final diff in either run mode.
8901    ///
8902    /// The summary is tied to the final commit and to the merge base of the
8903    /// resolved remote base; a record for any other pair is dropped and the
8904    /// agent asked again, so a description never outlives the diff it was
8905    /// written from. The seat is read-only and the prompt tells it to read the
8906    /// diff and the code around it. `[roles] synthesizer` selects the agents
8907    /// (unset: the default pick), each asked at most once, advancing on an
8908    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8909    /// carries the reason it is unavailable; nothing is invented then.
8910    async fn summarize_change(
8911        &mut self,
8912        winner: &Candidate,
8913        head: Option<&str>,
8914    ) -> std::result::Result<(), String> {
8915        let Some(head) = head else {
8916            return Err("the final commit could not be read".to_owned());
8917        };
8918        let remote = self.state.config.merge.remote.clone();
8919        let tracking = format!("{remote}/{}", self.state.base_branch);
8920        if !git::rev_exists(&winner.worktree, &tracking).await {
8921            return Err(format!("the remote base `{tracking}` could not be read"));
8922        }
8923        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8924            Ok(b) if !b.is_empty() => b,
8925            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8926        };
8927        let evidence = crate::pr_outcome::summary_evidence(
8928            &self.state,
8929            crate::pr_outcome::FinalHead::At(head),
8930        );
8931        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8932        if self
8933            .state
8934            .change_summary
8935            .as_ref()
8936            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8937        {
8938            return Ok(());
8939        }
8940        if self.state.change_summary.take().is_some() {
8941            self.state.event(
8942                "pr-summary",
8943                "the final diff or evidence changed; the earlier change summary was discarded",
8944            );
8945        }
8946        // Save the source snapshot even if no generation agent is available.
8947        if let Err(e) = write_artifact(
8948            &self.state,
8949            &format!("public-description-{fingerprint}.md"),
8950            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8951        ) {
8952            tracing::warn!("could not save public description evidence: {e:#}");
8953        }
8954        let chain = match agent::pick_chain(
8955            &self.state.config.agents,
8956            self.state.config.roles.synthesizer.as_ref(),
8957            &agent::installed,
8958            "synthesizer",
8959        ) {
8960            Ok(chain) => chain,
8961            Err(e) => {
8962                tracing::warn!("no agent for the change summary: {e:#}");
8963                self.state.event("pr-summary", "no summary agent available");
8964                return Err("no agent was available to summarize it".to_owned());
8965            }
8966        };
8967        let subjects = git::subjects(&winner.worktree, &base, head)
8968            .await
8969            .unwrap_or_default();
8970        let stat = git::diff_stat(&winner.worktree, &base, head)
8971            .await
8972            .unwrap_or_default();
8973        let stat: String = stat.chars().take(6_000).collect();
8974        let mut prompt = crate::prompt::change_summary_in(
8975            &base,
8976            head,
8977            &subjects,
8978            stat.trim(),
8979            &self.state.config.graph.github_language,
8980        );
8981        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8982        let sessions = self.state.config.graph.sessions;
8983        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8984        let prompts = self.state.config.prompts.clone();
8985        let cache = self.state.config.cache_dir();
8986        let run = self.state.id.clone();
8987        for (i, spec) in chain.into_iter().enumerate() {
8988            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8989            let github_spec = match agent::for_github_text(
8990                &spec,
8991                &self.state.config.graph.github_language,
8992                &winner.worktree,
8993            ) {
8994                Ok(spec) => spec,
8995                Err(e) => {
8996                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8997                    continue;
8998                }
8999            };
9000            let job = SeatJob {
9001                spec: github_spec,
9002                seat,
9003                cwd: winner.worktree.clone(),
9004                prompt: prompt.clone(),
9005                timeout,
9006                allow_write: false,
9007                sessions,
9008                artifacts: agent::artifacts_dir(&self.state.dir()),
9009                stem: if i == 0 {
9010                    "change-summary".to_owned()
9011                } else {
9012                    format!("change-summary-{}", spec.id)
9013                },
9014                handover: None,
9015            };
9016            let ctx = WaveCtx {
9017                run: &run,
9018                node: "pr-summary",
9019                prompts: &prompts,
9020                cache: cache.as_deref(),
9021                round: None,
9022                carry_seats: false,
9023                pauses: Some((&self.pause, &self.interrupt)),
9024            };
9025            let (seat, outcome) =
9026                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9027            self.state.seats.insert(seat.key.clone(), seat);
9028            let text = match outcome {
9029                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9030                AgentOutcome::ResourceBlocked(why) => {
9031                    // No agent was asked: the run stops resumably before
9032                    // anything is published (see `guarded_pr_message`).
9033                    if self.state.resource_block.is_none() {
9034                        self.state.resource_block = Some(crate::run::ResourceBlock {
9035                            cause: crate::run::ResourceBlockCause::Other,
9036                            node: "pr-summary".to_owned(),
9037                            kind: "shared resource".to_owned(),
9038                            detail: why,
9039                            at: Timestamp::now(),
9040                        });
9041                    }
9042                    return Err("a shared resource was busy".to_owned());
9043                }
9044                _ => continue,
9045            };
9046            let blind = &self.state.config.blind;
9047            let Some(title) = summary_title(&text) else {
9048                continue;
9049            };
9050            let body: String = summary_without_title(&text)
9051                .lines()
9052                .filter(|l| !l.contains("magi:run/"))
9053                .collect::<Vec<_>>()
9054                .join("\n");
9055            let body = blind::sanitize_prose(body.trim(), blind);
9056            let title = blind::sanitize_prose(&title, blind);
9057            if body.trim().is_empty() || title.trim().is_empty() {
9058                continue;
9059            }
9060            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9061                continue;
9062            }
9063            // The branch must still be the one the agent was shown.
9064            if git::rev_parse(&winner.worktree, "HEAD")
9065                .await
9066                .ok()
9067                .as_deref()
9068                != Some(head)
9069            {
9070                return Err("the branch moved while the summary was written".to_owned());
9071            }
9072            if git::merge_base(&winner.worktree, &tracking, head)
9073                .await
9074                .ok()
9075                .as_deref()
9076                != Some(base.as_str())
9077                || crate::pr_outcome::summary_evidence(
9078                    &self.state,
9079                    crate::pr_outcome::FinalHead::At(head),
9080                ) != evidence
9081            {
9082                return Err("the summary evidence changed while it was written".to_owned());
9083            }
9084            self.state.change_summary = Some(crate::run::ChangeSummary {
9085                head: head.to_owned(),
9086                base,
9087                evidence: fingerprint,
9088                title,
9089                body,
9090            });
9091            self.state.event(
9092                "pr-summary",
9093                format!("change summary written by {} for the final diff", spec.id),
9094            );
9095            if let Err(e) = self.state.save() {
9096                tracing::warn!("could not save the change summary: {e:#}");
9097            }
9098            return Ok(());
9099        }
9100        self.state
9101            .event("pr-summary", "no agent produced a change summary");
9102        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9103    }
9104
9105    /// One read-only rewrite request for a withheld description. The
9106    /// implementer's own conversation is tried first when it exists and can be
9107    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9108    /// writes it in a separate conversation, skipping the implementer's agent.
9109    /// The prompt carries the whole text, so no agent needs any memory. Each
9110    /// agent is asked at most once and the chain advances on an error, quota,
9111    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9112    /// that has run out of quota must not end the rewrite. Nothing is recorded
9113    /// as a handover.
9114    async fn rewrite_pr_text(
9115        &mut self,
9116        winner: &Candidate,
9117        pr: &PrMessage,
9118        violations: &[crate::github_text::Violation],
9119    ) -> Option<String> {
9120        let prompt = format!(
9121            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9122            pr.title, pr.body
9123        );
9124        let sessions = self.state.config.graph.sessions;
9125        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9126        // (spec, the resumed implementer's seat). A fallback seat is created
9127        // only when its turn comes, so an unused fallback leaves no record.
9128        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9129        let key = format!("impl-{}", winner.label);
9130        if let Some(spec) = self
9131            .state
9132            .config
9133            .agents
9134            .iter()
9135            .find(|a| a.id == winner.agent)
9136            .cloned()
9137            && let Some(existing) = self.state.seats.get(&key)
9138            && existing.agent == spec.id
9139            && agent::has_session(spec.kind, existing, sessions)
9140        {
9141            attempts.push((spec, Some(existing.clone())));
9142        }
9143        match agent::pick_chain(
9144            &self.state.config.agents,
9145            self.state.config.roles.language_judge.as_ref(),
9146            &agent::installed,
9147            "language_judge",
9148        ) {
9149            Ok(chain) => {
9150                for spec in chain {
9151                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9152                        continue;
9153                    }
9154                    attempts.push((spec, None));
9155                }
9156            }
9157            Err(e) if attempts.is_empty() => {
9158                tracing::warn!("no agent for the description rewrite: {e:#}");
9159                self.state
9160                    .event("github-text", "no rewrite agent available");
9161                return None;
9162            }
9163            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9164        }
9165        let prompts = self.state.config.prompts.clone();
9166        let cache = self.state.config.cache_dir();
9167        let run = self.state.id.clone();
9168        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9169            let resumed = resumed_seat.is_some();
9170            let seat = resumed_seat.unwrap_or_else(|| {
9171                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9172            });
9173            let github_spec = match agent::for_github_text(
9174                &spec,
9175                &self.state.config.graph.github_language,
9176                &winner.worktree,
9177            ) {
9178                Ok(spec) => spec,
9179                Err(e) => {
9180                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9181                    continue;
9182                }
9183            };
9184            let job = SeatJob {
9185                spec: github_spec,
9186                seat,
9187                cwd: winner.worktree.clone(),
9188                prompt: prompt.clone(),
9189                timeout: retry_budget(timeout_full, resumed),
9190                allow_write: false,
9191                sessions,
9192                artifacts: agent::artifacts_dir(&self.state.dir()),
9193                stem: if i == 0 {
9194                    "github-text-rewrite".to_owned()
9195                } else {
9196                    format!("github-text-rewrite-{}", spec.id)
9197                },
9198                handover: None,
9199            };
9200            let ctx = WaveCtx {
9201                run: &run,
9202                node: "github-text",
9203                prompts: &prompts,
9204                cache: cache.as_deref(),
9205                round: None,
9206                carry_seats: false,
9207                pauses: Some((&self.pause, &self.interrupt)),
9208            };
9209            let (seat, outcome) =
9210                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9211            self.state.seats.insert(seat.key.clone(), seat);
9212            match outcome {
9213                AgentOutcome::Ok(o)
9214                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9215                {
9216                    return Some(o.text);
9217                }
9218                AgentOutcome::ResourceBlocked(_) => return None,
9219                _ => {}
9220            }
9221        }
9222        None
9223    }
9224
9225    /// Abandon the standing posting-gate question, if any.
9226    fn retire_github_text_question(&mut self, why: &str) {
9227        let Some(id) = self
9228            .state
9229            .github_text
9230            .as_ref()
9231            .and_then(|g| g.question.clone())
9232        else {
9233            return;
9234        };
9235        let _ = ask::Questions::open().update(&id, |q| {
9236            q.abandon(why);
9237            Ok(())
9238        });
9239    }
9240
9241    /// Carry a withheld text to its end: file the question, park while it is
9242    /// open, and on an answer, silence or timeout pick the text to post. The
9243    /// decision is saved before it is used, so it applies once.
9244    async fn settle_withheld(
9245        &mut self,
9246        winner: &Candidate,
9247        pr: &PrMessage,
9248        mut g: crate::run::GithubTextAsk,
9249    ) -> Result<Option<PrMessage>> {
9250        use crate::github_text as gt;
9251        let store = ask::Questions::open();
9252        let timeout = self.state.config.graph.answer_timeout;
9253        if !g.resolved {
9254            let mut standing = g
9255                .question
9256                .as_ref()
9257                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9258            if let Some(q) = standing.as_ref()
9259                && q.status.open()
9260                && gt::expired(q, timeout)
9261            {
9262                let why = format!("no answer within {}s of asking", timeout.max(1));
9263                standing = store
9264                    .update(&q.id, |q| {
9265                        q.abandon(&why);
9266                        Ok(())
9267                    })
9268                    .ok()
9269                    .map(|(q, ())| q);
9270            }
9271            let mut ask_again: Option<Option<Vec<String>>> = None;
9272            match standing {
9273                // Recorded but never written (a stop between the two saves),
9274                // or removed: ask again without spending the retry.
9275                None if g.asks <= 1 => {
9276                    g.asks = 0;
9277                    ask_again = Some(None);
9278                }
9279                None => {
9280                    g.resolved = true;
9281                    g.chosen_title = None;
9282                    g.choice = Some(GithubTextChoice::Fallback);
9283                }
9284                Some(q) if q.status.open() => {
9285                    self.park_on_github_text(&g)?;
9286                    return Ok(None);
9287                }
9288                Some(q) => match gt::read_reply(&q) {
9289                    gt::Reply::Fallback => {
9290                        g.resolved = true;
9291                        g.chosen_title = None;
9292                        g.choice = Some(GithubTextChoice::Fallback);
9293                    }
9294                    gt::Reply::Original => {
9295                        g.resolved = true;
9296                        g.chosen_title = None;
9297                        g.choice = Some(GithubTextChoice::Original);
9298                    }
9299                    gt::Reply::Rewrite => {
9300                        g.resolved = true;
9301                        g.chosen_title = None;
9302                        g.choice = Some(if g.rewrite.is_some() {
9303                            GithubTextChoice::Rewrite
9304                        } else {
9305                            GithubTextChoice::Fallback
9306                        });
9307                    }
9308                    gt::Reply::Title(text) => {
9309                        let first = text
9310                            .lines()
9311                            .map(str::trim)
9312                            .find(|l| !l.is_empty())
9313                            .unwrap_or("");
9314                        let decision = if first.is_empty() {
9315                            None
9316                        } else {
9317                            let candidate = PrMessage {
9318                                title: first.to_owned(),
9319                                body: String::new(),
9320                            };
9321                            self.judge_pr_language(&winner.worktree, &candidate).await
9322                        };
9323                        match gt::vet_title(
9324                            &text,
9325                            decision,
9326                            self.state.config.graph.github_english(),
9327                        ) {
9328                            Ok(title) => {
9329                                g.resolved = true;
9330                                g.chosen_title = Some(title);
9331                                g.choice = Some(GithubTextChoice::MyTitle);
9332                            }
9333                            Err(cats) if g.asks < 2 => {
9334                                ask_again = Some(Some(
9335                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9336                                ))
9337                            }
9338                            Err(_) => {
9339                                g.resolved = true;
9340                                g.chosen_title = None;
9341                                g.choice = Some(GithubTextChoice::Fallback);
9342                            }
9343                        }
9344                    }
9345                },
9346            }
9347            if let Some(rejected) = ask_again {
9348                // A retry (after a rejected replacement title) is still about
9349                // the original text: the same ORIGINAL / FALLBACK comparison.
9350                let w = gt::Withheld {
9351                    title: g.title,
9352                    body: g.body,
9353                    categories: g.categories.clone(),
9354                };
9355                // Named by the original text's fingerprint alone, so a resume
9356                // rewrites the same file and the detail stays identical.
9357                let artifact = match crate::run::write_artifact(
9358                    &self.state,
9359                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9360                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9361                ) {
9362                    Ok(p) => Some(p),
9363                    Err(e) => {
9364                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9365                        None
9366                    }
9367                };
9368                let (detail, comparison) = gt::question_content(
9369                    &w,
9370                    &pr.title,
9371                    &pr.body,
9372                    &self.state.id,
9373                    rejected.as_deref(),
9374                    artifact.as_deref(),
9375                    g.rewrite.as_ref(),
9376                );
9377                let mut q = ask::Question::new(
9378                    self.state.id.clone(),
9379                    gt::ASK_NODE.to_owned(),
9380                    gt::ASK_SEAT.to_owned(),
9381                    gt::question_summary(&self.state.config.graph.language, &w),
9382                    detail,
9383                    gt::question_choices(&w, g.rewrite.as_ref()),
9384                );
9385                q.posting_comparison = Some(comparison);
9386                q.answer_timeout = timeout;
9387                // Recorded before it is written: a stop in between leaves a
9388                // record naming a question the resume then files again.
9389                g.question = Some(q.id.clone());
9390                g.asks += 1;
9391                self.state.github_text = Some(g.clone());
9392                self.state.event(
9393                    "github-text",
9394                    format!("asking the owner about the withheld text ({})", q.short()),
9395                );
9396                self.state.save()?;
9397                store
9398                    .put_executable(&mut q)
9399                    .context("file the posting-gate question")?;
9400                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9401                    tracing::warn!(
9402                        "could not notify about posting-gate question {}: {e:#}",
9403                        q.short()
9404                    );
9405                }
9406                self.park_on_github_text(&g)?;
9407                return Ok(None);
9408            }
9409        }
9410        let resolved = g.chosen_title.clone();
9411        let original = g.choice == Some(GithubTextChoice::Original);
9412        // Field names only: the event log never records the text.
9413        self.state.event(
9414            "github-text",
9415            if original {
9416                format!(
9417                    "posting the original text for: {}",
9418                    match (g.title, g.body) {
9419                        (true, true) => "title, description",
9420                        (true, false) => "title",
9421                        _ => "description",
9422                    }
9423                )
9424            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9425                "posting the saved rewrite title and description".to_owned()
9426            } else if resolved.is_some() {
9427                "posting the owner's replacement title".to_owned()
9428            } else {
9429                "posting the neutral text for the withheld fields".to_owned()
9430            },
9431        );
9432        self.state.github_text = Some(g.clone());
9433        self.state.save()?;
9434        if g.choice == Some(GithubTextChoice::Rewrite)
9435            && let Some(r) = g.rewrite.as_ref()
9436        {
9437            return Ok(Some(PrMessage {
9438                title: r.title.clone(),
9439                body: r.body.clone(),
9440            }));
9441        }
9442        let w = gt::Withheld {
9443            title: g.title,
9444            body: g.body,
9445            categories: g.categories.clone(),
9446        };
9447        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9448        if let Some(t) = resolved {
9449            title = t;
9450        }
9451        // Every withheld field is now neutral or vetted English.
9452        let english = gt::LanguageDecision {
9453            title_english: true,
9454            body_english: true,
9455            body_complete: true,
9456        };
9457        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9458        if original {
9459            // The owner's explicit choice overrides the gate for the fields it
9460            // was made for; the others keep what the gate produced.
9461            if g.title {
9462                title = pr.title.clone();
9463            }
9464            if g.body {
9465                body = pr.body.clone();
9466            }
9467        }
9468        Ok(Some(PrMessage { title, body }))
9469    }
9470
9471    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9472        self.state.github_text = Some(g.clone());
9473        self.state.end_pass_parked(ParkCause::Question);
9474        self.state.parked = true;
9475        self.state.event(
9476            "github-text",
9477            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9478        );
9479        self.state.save()
9480    }
9481
9482    /// Ask `[roles] language_judge` about this text and record which source
9483    /// decided; the text itself is never recorded.
9484    async fn judge_pr_language(
9485        &mut self,
9486        cwd: &Path,
9487        pr: &PrMessage,
9488    ) -> Option<crate::github_text::LanguageDecision> {
9489        self.state.config.roles.language_judge.as_ref()?;
9490        if !self.state.config.graph.language_gate() {
9491            return None;
9492        }
9493        let decision =
9494            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9495        self.state.event(
9496            "github-text",
9497            if decision.is_some() {
9498                "language decided by the language judge"
9499            } else {
9500                "language judge unavailable; using built-in heuristics"
9501            },
9502        );
9503        decision
9504    }
9505
9506    fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9507        Box::pin(self.merge_inner())
9508    }
9509
9510    async fn merge_inner(&mut self) -> Result<()> {
9511        if self.resource_halt()? || self.hold_base_sync().await? {
9512            return Ok(());
9513        }
9514        // Same reasoning as `gate`: ask the review and gate records directly
9515        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9516        // skip can rewrite on reentry to something that no longer says
9517        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9518        // so a hand-off (open findings, green verification) reaches merge
9519        // exactly like a genuinely clean round does.
9520        //
9521        // A run resumed mid-`land` never reaches here at all: `execute`
9522        // recognises `RunStatus::Landing` before it even calls `prep`, and
9523        // routes straight to `run_land` instead. That has to happen a level
9524        // up from this function, not with a check in here, because
9525        // `review_loop`'s own status recomputation (see its doc) runs
9526        // *before* `merge` on every reentry and would otherwise overwrite
9527        // the `Landing` marker with `Gating` before this node ever saw it.
9528        if self
9529            .state
9530            .base_sync
9531            .as_ref()
9532            .is_some_and(|s| s.conflict.is_some())
9533            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9534                != Some(RunStatus::Gating)
9535            // `gate_ran == false` is not "passed" - `gate` leaves it false
9536            // both before it has ever run and when its last attempt was
9537            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9538            // permission to merge on nothing but the review record. Only a
9539            // gate that actually ran may proceed; an intentional local
9540            // review can also finish with verification explicitly not configured.
9541            // `RunState::gate_status` computes that distinction.
9542            || !self.state.gate_allows_progress()
9543        {
9544            return Ok(());
9545        }
9546        // This node's own record, not `status`: `status == Ready` is not
9547        // unique to the harmless `MergeMode::None` path this line was
9548        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9549        // run's PR was closed without merging — and on that run `mode` is
9550        // still `Pr`, so a reentry that fell through here would push and
9551        // open a second pull request. `self.state.merge` is set exactly once
9552        // this node (or `land`) has already produced a verdict, under every
9553        // mode, which is what "already done" actually means here.
9554        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9555            return Ok(());
9556        }
9557        let Some(winner) = self.state.winner().cloned() else {
9558            return Ok(());
9559        };
9560        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9561        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9562            self.state.status = RunStatus::Blocked;
9563            self.state.event(
9564                "gate",
9565                "winner changed after gate; fresh verification required",
9566            );
9567            self.state.save()?;
9568            return Ok(());
9569        }
9570        let wait = self.repository_wait();
9571        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9572            return Ok(());
9573        }
9574        let repo = self.state.repo.clone();
9575        let base = self.state.base_branch.clone();
9576        let mode = self.state.config.merge.mode;
9577        let style = self.state.config.merge.style;
9578        let facts = if is_review_run(&self.state) {
9579            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9580            let start = review_base(
9581                &repo,
9582                &self.state.config.merge.remote,
9583                &base,
9584                &self.state.base_commit,
9585                &winner.branch,
9586            )
9587            .await;
9588            branch_facts(&repo, &start, &winner.branch).await
9589        } else {
9590            None
9591        };
9592        // `None` when the base could not be freshly read: then an adopted
9593        // pull request's title is left alone.
9594        let leaked = if is_review_run(&self.state) {
9595            leaked_subjects(&self.state, &winner.branch).await
9596        } else {
9597            Some(Vec::new())
9598        };
9599        // `None`: parked on the owner's word about a withheld title or body.
9600        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9601        // the resume arrives here again.
9602        let Some(pr) = self
9603            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9604            .await?
9605        else {
9606            return Ok(());
9607        };
9608        let message = pr.commit_message();
9609
9610        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9611        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9612            return Ok(());
9613        }
9614        let outcome = match mode {
9615            MergeMode::None => MergeOutcome {
9616                mode,
9617                ok: true,
9618                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9619                empty: false,
9620            },
9621            MergeMode::Pr | MergeMode::Local
9622                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9623            {
9624                MergeOutcome {
9625                    mode,
9626                    ok: false,
9627                    detail: empty_candidate_detail(&self.state, &base),
9628                    empty: true,
9629                }
9630            }
9631            MergeMode::Local => {
9632                let on = git::current_branch(&repo).await?;
9633                if on.as_deref() != Some(base.as_str()) {
9634                    MergeOutcome {
9635                        mode,
9636                        ok: false,
9637                        detail: format!(
9638                            "{} has {} checked out, not the base branch {base}",
9639                            repo.display(),
9640                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9641                        ),
9642                        empty: false,
9643                    }
9644                } else if !git::is_clean(&repo).await? {
9645                    MergeOutcome {
9646                        mode,
9647                        ok: false,
9648                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9649                        empty: false,
9650                    }
9651                } else {
9652                    let out = match style {
9653                        MergeStyle::Merge => {
9654                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9655                        }
9656                        MergeStyle::Squash => {
9657                            git::merge_squash(&repo, &winner.branch, &message).await?
9658                        }
9659                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9660                    };
9661                    MergeOutcome {
9662                        mode,
9663                        ok: out.ok(),
9664                        detail: if out.ok() { out.stdout } else { out.stderr },
9665                        empty: false,
9666                    }
9667                }
9668            }
9669            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9670                self.state.pr_publication_pending = true;
9671                self.state.save()?;
9672                let out =
9673                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9674                self.state.pr_publication_pending = out.is_err();
9675                MergeOutcome {
9676                    mode,
9677                    ok: out.is_ok(),
9678                    detail: match out {
9679                        Ok(url) => url,
9680                        Err(e) => format!("{e:#}"),
9681                    },
9682                    empty: false,
9683                }
9684            }
9685            MergeMode::Pr => {
9686                let remote = self.state.config.merge.remote.clone();
9687                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9688                let pushed = if self.state.foreign_checkout.is_some() {
9689                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9690                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9691                } else {
9692                    git::push(&winner.worktree, &remote, &winner.branch).await?
9693                };
9694                if !pushed.ok() {
9695                    MergeOutcome {
9696                        mode,
9697                        ok: false,
9698                        detail: pushed.stderr,
9699                        empty: false,
9700                    }
9701                } else {
9702                    if self.state.foreign_checkout.is_some() {
9703                        self.state.pr_pushed_head = Some(delivery_head.clone());
9704                    }
9705                    // A retry or resume of a run whose branch already has an
9706                    // open pull request adopts it rather than failing on a
9707                    // duplicate. Only this winner branch into this base:
9708                    // `branch_for` derives the name from the run id, so a
9709                    // different run's pull request never matches.
9710                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9711                    let out = match pr_merge_plan(found) {
9712                        PrPlan::Create => {
9713                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9714                                .await
9715                        }
9716                        PrPlan::Adopt { url, title } => {
9717                            self.state
9718                                .event("merge", format!("Pr: adopted open pull request {url}"));
9719                            if self.state.explicit_github_text_selection() {
9720                                // An explicit publication selection applies the prepared
9721                                // payload; acceptance alone is not publication proof.
9722                                land::set_pr_text_verbatim(
9723                                    &winner.worktree,
9724                                    &url,
9725                                    &pr.title,
9726                                    &pr.body,
9727                                )
9728                                .await
9729                                .map(|()| url)
9730                            } else {
9731                                if title != pr.title
9732                                    && (!is_review_run(&self.state)
9733                                        || leaked
9734                                            .as_deref()
9735                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9736                                    && let Err(e) =
9737                                        if self.state.github_text.as_ref().is_some_and(|g| {
9738                                            g.resolved
9739                                                && g.title
9740                                                && g.choice == Some(GithubTextChoice::Original)
9741                                        }) {
9742                                            // The owner chose to post this title unmodified.
9743                                            land::set_pr_title_verbatim(
9744                                                &winner.worktree,
9745                                                &url,
9746                                                &pr.title,
9747                                            )
9748                                            .await
9749                                        } else {
9750                                            land::set_pr_title(
9751                                                &mut self.state,
9752                                                &winner.worktree,
9753                                                &url,
9754                                                &pr.title,
9755                                            )
9756                                            .await
9757                                        }
9758                                {
9759                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9760                                    self.state
9761                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9762                                }
9763                                Ok(url)
9764                            }
9765                        }
9766                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9767                    };
9768                    match out {
9769                        Ok(url) => MergeOutcome {
9770                            mode,
9771                            ok: true,
9772                            detail: url,
9773                            empty: false,
9774                        },
9775                        Err(e) => MergeOutcome {
9776                            mode,
9777                            ok: false,
9778                            detail: e.to_string(),
9779                            empty: false,
9780                        },
9781                    }
9782                }
9783            }
9784        };
9785
9786        self.state.status = match (mode, outcome.ok) {
9787            (MergeMode::None, _) => RunStatus::Ready,
9788            (_, true) if outcome.empty => RunStatus::Ready,
9789            (MergeMode::Pr, true) => RunStatus::Ready,
9790            (_, true) => RunStatus::Merged,
9791            (_, false) => RunStatus::Blocked,
9792        };
9793        self.state.event(
9794            "merge",
9795            format!(
9796                "{:?}: {}",
9797                mode,
9798                outcome.detail.lines().next().unwrap_or("")
9799            ),
9800        );
9801        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9802            self.state.record_github_text_publication(&outcome.detail);
9803        }
9804        self.state.merge = Some(outcome);
9805        self.state.save()?;
9806
9807        // The PR is open and the run would historically stop here, leaving the
9808        // operator to watch checks, feed review comments back to a fixer, and
9809        // merge. That was done by hand six times in one session before this
9810        // existed. Opt-in, because merging is the one irreversible thing magi
9811        // can do to a repository.
9812        if self.state.config.graph.land
9813            && mode == MergeMode::Pr
9814            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9815        {
9816            self.run_land().await?;
9817        }
9818        // `run_land` may have left `status` at `Landing` - still waiting on
9819        // CI or the owner's approval, not actually settled - so this has to
9820        // read whatever `status` ended up as here, not the `Merged` this
9821        // function set a few lines up.
9822        self.settle_questions();
9823        Ok(())
9824    }
9825
9826    /// Enter `land`.
9827    ///
9828    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9829    /// a resumed run's re-entry. `land::land` itself is what serialises the
9830    /// two git-mutating moments inside the loop — the rebase push and
9831    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9832    /// to hold a lock across the whole call, and doing so would serialise
9833    /// this run's CI wait against a *different* run's land-approval resume
9834    /// in the same repository, which is exactly the "must not wait on
9835    /// another task" property the daemon's slot-freeing exists to give.
9836    fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9837        Box::pin(self.run_land_inner())
9838    }
9839
9840    async fn run_land_inner(&mut self) -> Result<()> {
9841        crate::pr_head::ensure_owner(&self.state)?;
9842        let url = self
9843            .state
9844            .pr
9845            .as_ref()
9846            .map(|pr| pr.url.clone())
9847            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9848            .unwrap_or_default();
9849        let url = url.lines().next().unwrap_or("").trim().to_owned();
9850        if !url.starts_with("http") {
9851            return Ok(());
9852        }
9853        // A land failure is not a lost run: the work is on a branch and the
9854        // pull request is open, which is exactly where a human takes over.
9855        loop {
9856            match land::land(&mut self.state, &url).await {
9857                Ok(_)
9858                    if self
9859                        .state
9860                        .merge_repairs
9861                        .last()
9862                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9863                {
9864                    if self.repair_merge_review().await? {
9865                        continue;
9866                    }
9867                }
9868                Ok(_) if self.state.resource_block.is_some() => {
9869                    return self.stop_for_resource();
9870                }
9871                Ok(pr) if self.state.parked => {
9872                    // `land` already saved the parked marker; nothing here
9873                    // overrides `status` back to a terminal value while an
9874                    // approval is still outstanding.
9875                    let _ = pr;
9876                }
9877                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9878                    self.state.status = RunStatus::Superseded;
9879                    self.state.save()?;
9880                }
9881                Ok(pr) => {
9882                    self.state.status = match pr.state {
9883                        land::PrLifecycle::Merged => RunStatus::Merged,
9884                        _ => RunStatus::Blocked,
9885                    };
9886                    // Downstream of a confirmed merge only - see
9887                    // `bump::should_release_bump`'s own doc for why this one
9888                    // check covers all three of `land`'s success paths.
9889                    // Best-effort: the run already landed, so a failure here
9890                    // (the decision call, `gh`, `cargo`) is recorded and never
9891                    // turns a landed run into a failed one.
9892                    if bump::should_release_bump(self.state.status)
9893                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9894                    {
9895                        // The event is the run's own record. Not-eligible cases
9896                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9897                        // `Err` is a bump that was tried and failed:
9898                        // `after_merge` itself raises the operator notice for
9899                        // that, whether or not a release PR exists yet.
9900                        self.state
9901                            .event("bump", format!("release bump skipped: {e:#}"));
9902                    }
9903                    // Independent of the bump, and best-effort in the same way:
9904                    // findings the merge left open become follow-up tasks.
9905                    if self.state.status == RunStatus::Merged {
9906                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9907                    }
9908                    self.state.save()?;
9909                }
9910                Err(e) => {
9911                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9912                        return Err(e);
9913                    }
9914                    self.state.status = RunStatus::Blocked;
9915                    self.state.event("land", format!("gave up: {e}"));
9916                    self.state.save()?;
9917                }
9918            }
9919            break;
9920        }
9921        Ok(())
9922    }
9923
9924    /// One repair cycle stays under the existing driver/task claim. An
9925    /// interrupted fixer is rolled back and held rather than run twice.
9926    async fn repair_merge_review(&mut self) -> Result<bool> {
9927        use crate::run::MergeRepairPhase as Phase;
9928        crate::pr_head::ensure_owner(&self.state)?;
9929        let i = self.state.merge_repairs.len() - 1;
9930        let request = self.state.merge_repairs[i].clone();
9931        if request.phase == Phase::Fixing {
9932            let restored = match self.state.winner().cloned() {
9933                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9934                None => Err(anyhow::anyhow!("repair has no winning branch")),
9935            };
9936            let why = match restored {
9937                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9938                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9939                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9940            };
9941            self.hold_merge_repair(i, why)?;
9942            return Ok(false);
9943        }
9944        // A completed repair gate is a publication checkpoint. On reentry,
9945        // reject external changes before the ordinary gate path can replace
9946        // that evidence by verifying a new head. Requested repairs still
9947        // start their own review generation below.
9948        if request.phase == Phase::Reviewing && self.state.gate_ran {
9949            let winner = self
9950                .state
9951                .winner()
9952                .context("repair has no winning branch")?;
9953            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9954            if self.state.gate_head.as_deref() != Some(head.as_str())
9955                || !git::is_clean(&winner.worktree).await?
9956            {
9957                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9958                return Ok(false);
9959            }
9960        }
9961        let failure = if i >= self.state.config.graph.land_rounds
9962            || self.state.config.graph.review_rounds == 0
9963        {
9964            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9965        } else {
9966            None
9967        };
9968        if let Some(why) = failure {
9969            self.hold_merge_repair(i, why)?;
9970            return Ok(false);
9971        }
9972        if request.phase == Phase::Requested {
9973            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9974                Ok(pr) => pr,
9975                Err(e) => {
9976                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9977                    return Ok(false);
9978                }
9979            };
9980            self.state.merge_repairs[i].phase = Phase::Fixing;
9981            self.state.land_approval = None;
9982            self.state.land_armed_head = None;
9983            self.state.save()?;
9984            // Serialize the exact records, including finding ids/details,
9985            // reconsideration reasons, final votes and verification output.
9986            let evidence = serde_json::to_string_pretty(&request)?;
9987            let reason = format!(
9988                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9989            );
9990            let budget = self.state.config.graph.land_rounds;
9991            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9992            match result {
9993                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9994                Ok(land::Fixed::Deferred) => {
9995                    // An unavailable fixer did not spend this repair request.
9996                    // Restore partial fallback work before retrying it, as for
9997                    // an interrupted repair, without closing the request.
9998                    let worktree = self
9999                        .state
10000                        .winner()
10001                        .context("repair has no winning branch")?
10002                        .worktree
10003                        .clone();
10004                    self.graft_restore(&worktree, &request.head).await?;
10005                    self.state.merge_repairs[i].phase = Phase::Requested;
10006                    self.stop_for_resource()?;
10007                    return Ok(false);
10008                }
10009                Ok(land::Fixed::Failed(why)) => {
10010                    self.hold_merge_repair(
10011                        i,
10012                        format!(
10013                            "repair failed: {why}; inspect the branch and run an explicit review"
10014                        ),
10015                    )?;
10016                    return Ok(false);
10017                }
10018                Err(e) => {
10019                    self.hold_merge_repair(
10020                        i,
10021                        format!(
10022                            "repair failed: {e:#}; inspect the branch and run an explicit review"
10023                        ),
10024                    )?;
10025                    return Ok(false);
10026                }
10027            }
10028            // Even no change gets a real new reviewer wave. The previous
10029            // evidence remains in the request; a former clean/concluded round
10030            // cannot short-circuit the new review or gate.
10031            self.begin_merge_repair_review(i)?;
10032        }
10033        self.review_loop().await?;
10034        self.gate().await?;
10035        if self.resource_halt()? || self.state.parked {
10036            return Ok(false);
10037        }
10038        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10039            != Some(RunStatus::Gating)
10040            || !self.state.gate_allows_progress()
10041        {
10042            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10043            return Ok(false);
10044        }
10045        // Review and gate may have added commits after the initial repair.
10046        // Publish exactly their final head to the existing PR, with a lease.
10047        let winner = self
10048            .state
10049            .winner()
10050            .cloned()
10051            .context("repair has no winning branch")?;
10052        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10053        if self.state.gate_head.as_deref() != Some(after.as_str())
10054            || !git::is_clean(&winner.worktree).await?
10055        {
10056            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10057            return Ok(false);
10058        }
10059        let expected = self
10060            .state
10061            .pr_pushed_head
10062            .as_deref()
10063            .unwrap_or(&request.head)
10064            .to_owned();
10065        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10066        // Push the checked commit itself: a local ref moving after the check
10067        // must not change which commit the forge receives.
10068        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10069            return Ok(false);
10070        }
10071        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10072        let refspec = format!("{after}:refs/heads/{public}");
10073        let push = git::git_raw(
10074            &winner.worktree,
10075            &["push", &lease, &self.state.config.merge.remote, &refspec],
10076        )
10077        .await;
10078        match push {
10079            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10080            Ok(out) => {
10081                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10082                return Ok(false);
10083            }
10084            Err(e) => {
10085                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10086                return Ok(false);
10087            }
10088        }
10089        self.state.merge_repairs[i].phase = Phase::Complete;
10090        self.state.status = RunStatus::Landing;
10091        self.state.event(
10092            "repair",
10093            "new review and gate completed; fresh owner merge decision required",
10094        );
10095        self.state.save()?;
10096        Ok(true)
10097    }
10098
10099    fn merge_repair_artifacts(&self) -> PathBuf {
10100        let artifacts = agent::artifacts_dir(&self.state.dir());
10101        match self
10102            .state
10103            .merge_repairs
10104            .last()
10105            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10106        {
10107            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10108            None => artifacts,
10109        }
10110    }
10111
10112    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10113        if let Some(e2e) = self.state.head_e2e.take() {
10114            self.state
10115                .verification_history
10116                .push(crate::run::VerificationEvidence {
10117                    superseded_by: None,
10118                    archived_at: Some(Timestamp::now()),
10119                    fixes_spent: Some(self.state.gate_fixes.len()),
10120                    gate_head: None,
10121                    gate_ran: false,
10122                    gate: Vec::new(),
10123                    e2e: Some(e2e),
10124                });
10125        }
10126        self.state.reviews.clear();
10127        self.state.gate.clear();
10128        self.state.gate_ran = false;
10129        self.state.gate_head = None;
10130        self.state.gate_fixes.clear();
10131        self.state.gate_repair_e2e = None;
10132        self.state.pre_gate.clear();
10133        self.state.pre_gate_commit = None;
10134        self.state.contested_handoff = None;
10135        self.state.land_reruns = None;
10136        self.state.land_approval = None;
10137        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10138        self.state.status = RunStatus::Reviewing;
10139        self.state.save()
10140    }
10141
10142    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10143        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10144        self.state.merge_repairs[index].failure = Some(why.clone());
10145        self.state.status = RunStatus::Blocked;
10146        self.state.event("repair", why);
10147        self.state.save()
10148    }
10149
10150    // -------------------------------------------------------------- helpers
10151
10152    /// Fetch or create a seat, keeping its conversation across nodes.
10153    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10154        if let Some(existing) = self.state.seats.get(key)
10155            && existing.agent == agent
10156        {
10157            return existing.clone();
10158        }
10159        // A seat that changes agent mints its session id from the agent too,
10160        // like a handover: the old agent's uuid is already taken by the CLI.
10161        let fresh = if self.state.seats.contains_key(key) {
10162            handover_seat(key, agent, self.state.next_seat_seed())
10163        } else {
10164            SeatState::new(key, agent, self.state.seed)
10165        };
10166        self.state.seats.insert(key.to_owned(), fresh.clone());
10167        fresh
10168    }
10169
10170    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10171    /// seat to another roster agent, in which case that agent. Nodes that
10172    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10173    /// reconsideration) must keep talking to whoever answered it, not slip
10174    /// back to the agent that failed it.
10175    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10176        match self.state.seats.get(key) {
10177            Some(s) if s.agent != spec.id => {
10178                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10179            }
10180            _ => spec,
10181        }
10182    }
10183
10184    /// A candidate rendered for judging, with the leak policy applied.
10185    fn view(&self, c: &Candidate) -> CandidateView {
10186        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10187            .unwrap_or_default();
10188        let (patch, _) = blind::sanitize_patch(
10189            &format!("candidate {} patch", c.label),
10190            &raw,
10191            &self.state.config.blind,
10192        );
10193        CandidateView {
10194            label: c.label,
10195            branch: c.branch.clone(),
10196            summary: c.summary.clone(),
10197            stat: c.stat.clone(),
10198            patch,
10199        }
10200    }
10201
10202    /// The full candidate set as prompt text, for seats with no live session.
10203    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10204        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10205        prompt::judge(
10206            "(see above)",
10207            &views,
10208            self.roles.judges.len(),
10209            base_short,
10210            "en",
10211        )
10212    }
10213
10214    /// The final-vote prompt with everything a seat that has no session of its
10215    /// own needs: the candidates, the seat's own ranking and reasons, and the
10216    /// anonymised deliberation it took part in (only when there was one, so a
10217    /// handed-over seat never sees more than the seat it replaces did). The
10218    /// `Final vote` heading stays first.
10219    fn vote_prompt_full(
10220        &self,
10221        j: usize,
10222        labels: &[char],
10223        language: &str,
10224        candidates: &[Candidate],
10225        base_short: &str,
10226    ) -> String {
10227        let mut text = format!(
10228            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10229            prompt::final_vote(labels, language),
10230            self.state.instruction,
10231            self.candidate_block(candidates, base_short)
10232        );
10233        if let Some(own) = self
10234            .state
10235            .judgements
10236            .get(j)
10237            .filter(|r| !r.ranking.is_empty())
10238        {
10239            let reasons = own
10240                .reasons
10241                .iter()
10242                .map(|(k, v)| format!("- {k}: {v}"))
10243                .collect::<Vec<_>>()
10244                .join("\n");
10245            text.push_str(&format!(
10246                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10247                own.ranking.iter().collect::<String>(),
10248                if reasons.is_empty() {
10249                    ""
10250                } else {
10251                    ", because:\n"
10252                }
10253            ));
10254        }
10255        if !self.state.deliberation.is_empty() {
10256            text.push_str("\n# What was argued before this vote\n");
10257            for t in self.transcript(&[], j) {
10258                text.push_str(&format!(
10259                    "\n## {}{}\n\n{}\n",
10260                    t.who,
10261                    if t.is_self { " (you)" } else { "" },
10262                    t.body.trim()
10263                ));
10264            }
10265        }
10266        text
10267    }
10268
10269    /// Anonymised transcript for judge `self_idx`.
10270    ///
10271    /// The initial rankings are always the opening statements. Seeding them
10272    /// only when no turn had been taken yet meant every judge after the first
10273    /// argued against a single voice instead of against the actual split — the
10274    /// disagreement is the information, so it is always on the table.
10275    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10276        let mut turns = Vec::new();
10277        for j in &self.state.judgements {
10278            if j.ranking.is_empty() {
10279                continue;
10280            }
10281            let reasons = j
10282                .reasons
10283                .iter()
10284                .map(|(k, v)| format!("- {k}: {v}"))
10285                .collect::<Vec<_>>()
10286                .join("\n");
10287            turns.push(Turn {
10288                who: format!("Judge {} (opening ranking)", j.judge),
10289                is_self: j.judge == self_idx + 1,
10290                body: format!(
10291                    "Ranked {}{}{reasons}",
10292                    j.ranking.iter().collect::<String>(),
10293                    if reasons.is_empty() {
10294                        ""
10295                    } else {
10296                        ", because:\n"
10297                    }
10298                ),
10299            });
10300        }
10301        for t in self
10302            .state
10303            .deliberation
10304            .iter()
10305            .flat_map(|r| r.turns.iter())
10306            .chain(current)
10307        {
10308            turns.push(Turn {
10309                who: format!("Judge {}", t.judge),
10310                is_self: t.judge == self_idx + 1,
10311                body: t.body.clone(),
10312            });
10313        }
10314        turns
10315    }
10316}
10317
10318/// Does this seat still hold the context a follow-up prompt would rely on?
10319fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10320    agent::has_session(spec.kind, seat, sessions)
10321}
10322
10323/// The next entry in `roster` after `start`, never wrapping back to the
10324/// front, whose id is not in `tried` yet.
10325///
10326/// Starts one past `start` rather than at the front of `roster`: `start` is
10327/// the seat's own original position, and a seat whose candidate slot already
10328/// sits on the roster's second entry must fall through to the third next, not
10329/// restart at the first — which is very likely a different candidate's own
10330/// agent already. Never wraps back past `start`, for the same reason: an
10331/// entry earlier in the roster than the seat's own position is almost
10332/// certainly some *other* candidate slot's own agent, and once the tail of
10333/// the roster is exhausted there are no more untried agents for *this* seat
10334/// to fall through to — the caller's fallback chain ends there, exactly as
10335/// "no further untried agents remain in the list for that seat" asks for.
10336///
10337/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10338/// the same id twice (an operator's `roles.implementers` typo, or a
10339/// `[[agents]]` list reused across roles) must not let
10340/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10341/// over `roster` either finds an untried id or runs out, so this always
10342/// terminates regardless of duplicates.
10343fn next_untried_in_roster<'a>(
10344    roster: &'a [AgentSpec],
10345    start: usize,
10346    tried: &BTreeSet<String>,
10347) -> Option<&'a AgentSpec> {
10348    roster
10349        .get(start + 1..)?
10350        .iter()
10351        .find(|s| !tried.contains(&s.id))
10352}
10353
10354/// The successor for a seat, shared by all four seat kinds (implement, judge,
10355/// review, advise). `others` holds the ids that *currently* occupy the other
10356/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10357/// sees them), so roster entries beyond the seat count act as spares: a failed
10358/// seat goes to an agent no other seat holds whenever the roster permits.
10359///
10360/// `carried` is `Some` only for the review loop's carried failure history.
10361/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10362/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10363/// whole roster: untried and not another seat's occupant; (3) the plain walk
10364/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10365/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10366/// a better outcome than an empty seat, and it keeps the answer to "is there
10367/// a successor at all" exactly what it was before occupants were considered,
10368/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10369/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10370/// carried failure is a cheaper bet than doubling an agent on the panel, and
10371/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10372/// the same round are handled one at a time, so a seat later in the batch
10373/// sees an earlier one's new occupant but not yet its own freed agent.
10374fn pick_successor<'a>(
10375    roster: &'a [AgentSpec],
10376    start: usize,
10377    tried: &BTreeSet<String>,
10378    carried: Option<&BTreeSet<String>>,
10379    others: &BTreeSet<String>,
10380) -> Option<&'a AgentSpec> {
10381    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10382    roster
10383        .get(start + 1..)
10384        .and_then(|tail| {
10385            tail.iter()
10386                .filter(free)
10387                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10388        })
10389        .or_else(|| {
10390            carried
10391                .is_some()
10392                .then(|| roster.iter().find(free))
10393                .flatten()
10394        })
10395        .or_else(|| match carried {
10396            Some(c) => next_for_seat(roster, start, tried, c),
10397            None => next_untried_in_roster(roster, start, tried),
10398        })
10399}
10400
10401/// The next agent for a seat that carries its failure history across rounds
10402/// (the review loop). `round_tried` is this round's own bound and starts
10403/// empty every round; `carried_failed` only decides priority.
10404///
10405/// First: an id walking forward from `start`, never wrapping, that is neither
10406/// tried this round nor failed in an earlier one. Only when that is exhausted
10407/// does it rescue: the first roster id (in roster order, so this one *does*
10408/// look before `start`) not yet tried this round, which is by then a carried
10409/// failure. Each id is rescued at most once per round, so it cannot loop.
10410fn next_for_seat<'a>(
10411    roster: &'a [AgentSpec],
10412    start: usize,
10413    round_tried: &BTreeSet<String>,
10414    carried_failed: &BTreeSet<String>,
10415) -> Option<&'a AgentSpec> {
10416    roster
10417        .get(start + 1..)?
10418        .iter()
10419        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10420        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10421}
10422
10423/// Where a reviewer seat starts a round: the agent that last answered it when
10424/// it is still on the roster and not marked failed, else the spec's own agent
10425/// unless it failed, else the next roster agent that has not failed, else the
10426/// spec's own agent again (the whole roster failed). Ids no longer on the
10427/// roster are ignored. An empty roster has no handover, so the spec stands.
10428fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10429    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10430        return spec;
10431    };
10432    let ok = |id: &str| !h.failed.contains(id);
10433    if let Some(last) = h.last_ok.as_deref()
10434        && ok(last)
10435        && let Some(s) = roster.iter().find(|s| s.id == last)
10436    {
10437        return s.clone();
10438    }
10439    if ok(&spec.id) {
10440        return spec;
10441    }
10442    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10443    roster
10444        .get(from + 1..)
10445        .into_iter()
10446        .flatten()
10447        .chain(roster.iter())
10448        .find(|s| ok(&s.id))
10449        .cloned()
10450        .unwrap_or(spec)
10451}
10452
10453/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10454/// seed so a CLI that mints its session id up front (`--session-id`) never
10455/// reuses the uuid the previous agent already opened under the same seat key.
10456pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10457    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10458}
10459
10460/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10461/// so the classifier below and the code that builds the message cannot drift.
10462const TIMED_OUT: &str = "timed out";
10463
10464impl FailClass {
10465    /// `None` for an answer; otherwise how the turn failed.
10466    fn of(out: &AgentOutcome) -> Option<Self> {
10467        match out {
10468            AgentOutcome::Ok(_) => None,
10469            AgentOutcome::Quota(_) => Some(Self::Quota),
10470            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10471            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10472            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10473            // Not a failure of the agent. Callers check for it first; `None`
10474            // here means "no class to hand over on".
10475            AgentOutcome::ResourceBlocked(_) => None,
10476        }
10477    }
10478
10479    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10480    fn stem_word(&self) -> &'static str {
10481        match self {
10482            Self::Quota => "quota",
10483            _ => "handover",
10484        }
10485    }
10486}
10487
10488/// The message's first line with its variable parts removed — digit runs and
10489/// path-like tokens — so "exit code 2" and "exit code 7" read
10490/// as one kind of failure.
10491fn failure_signature(msg: &str) -> String {
10492    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10493    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10494    // the historical machine key while the process diagnosis uses readable prose.
10495    if let Some(code) = line
10496        .strip_prefix("process exited with no usable output (exit code ")
10497        .and_then(|s| s.strip_suffix(')'))
10498        .and_then(|s| s.parse::<i32>().ok())
10499    {
10500        line = format!("exited with some({code}) and no usable output");
10501    } else if matches!(
10502        line.as_str(),
10503        "process exited with no usable output (no exit code reported)"
10504            | "process exited with no usable output (exit code unavailable)"
10505    ) {
10506        line = "exited with none and no usable output".to_owned();
10507    }
10508    let mut out = Vec::new();
10509    for word in line.split_whitespace() {
10510        if word.contains('/') || word.contains('\\') {
10511            out.push("<path>".to_owned());
10512            continue;
10513        }
10514        let mut w = String::new();
10515        let mut in_digits = false;
10516        for c in word.chars() {
10517            if c.is_ascii_digit() {
10518                if !in_digits {
10519                    w.push('#');
10520                }
10521                in_digits = true;
10522            } else {
10523                in_digits = false;
10524                w.push(c);
10525            }
10526        }
10527        out.push(w);
10528    }
10529    out.join(" ").chars().take(120).collect()
10530}
10531
10532/// Whether a seat that just failed with `cur` may go to the next roster agent.
10533/// A quota or a timeout always may. Any other failure may not when the agent
10534/// before it failed the same way: an error the prompt causes would otherwise
10535/// walk the whole roster. `prev` is the class of the immediately preceding
10536/// agent's failure, so a quota or timeout in between breaks the run of
10537/// identical failures by itself.
10538fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10539    match cur {
10540        FailClass::Quota | FailClass::Timeout => true,
10541        FailClass::Other(_) => prev != Some(cur),
10542    }
10543}
10544
10545/// A short human reason for a failed outcome, for the handover record.
10546fn fail_reason(out: &AgentOutcome) -> String {
10547    match out {
10548        AgentOutcome::Ok(_) => String::new(),
10549        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10550        AgentOutcome::Dropped(o) => format!(
10551            "the CLI dropped the stream ({})",
10552            o.dropped
10553                .as_ref()
10554                .map(|d| d.why.as_str())
10555                .unwrap_or("it ended without delivering its answer")
10556        ),
10557        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10558            e.lines().next().unwrap_or("").chars().take(160).collect()
10559        }
10560    }
10561}
10562
10563/// Note one handover in the run: the structured record and, in the timeline,
10564/// the sentence a person reads. A quota keeps the wording it always had.
10565pub(crate) fn record_handover(
10566    state: &mut RunState,
10567    node: &str,
10568    seat: &str,
10569    from: &str,
10570    to: &str,
10571    class: &FailClass,
10572    reason: &str,
10573) {
10574    let message = if *class == FailClass::Quota {
10575        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10576    } else {
10577        format!("{seat}: handed over {from} -> {to} ({reason})")
10578    };
10579    state.event(node, message);
10580    state.handovers.push(Handover {
10581        at: Timestamp::now(),
10582        node: node.to_owned(),
10583        seat: seat.to_owned(),
10584        from: from.to_owned(),
10585        to: to.to_owned(),
10586        reason: reason.to_owned(),
10587    });
10588}
10589
10590/// Did this reply report running a command whose own CLI never confirmed an
10591/// exit status?
10592///
10593/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10594/// command *finished* (see that type's own doc), so this can only be `true`
10595/// for a command whose completion event carried no readable exit code — not
10596/// for one that simply is not mentioned at all. That is the one signal this
10597/// crate can read, from the same record `state.jobs` renders, about a reply
10598/// standing next to work its own CLI cannot vouch for finishing; it is
10599/// deliberately not a check on the exit code's *value* (a fixer legitimately
10600/// runs a command that fails mid-iteration before it succeeds) and not a
10601/// guess at a command still running in the background (which emits no event
10602/// at all, and so leaves no evidence here to find).
10603fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10604    commands.iter().any(|c| c.exit_code.is_none())
10605}
10606
10607/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10608/// trusted as a verified no-op — the adoption guard's own text-level half.
10609///
10610/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10611/// timed out): a marker only earns the benefit of the doubt from a turn the
10612/// CLI itself vouches for finishing properly, the same house style
10613/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10614/// *fix* report to for `commands`. A candidate that timed out, exited
10615/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10616/// is, whatever prose it wrote — this returns `None` before it ever looks at
10617/// `text`. The remaining guards (the tree really is empty, the evidence is
10618/// non-empty) are the caller's: this only reads what the reply *claimed*.
10619fn verified_noop_claim(
10620    usable: bool,
10621    commands: &[agent::CommandEvidence],
10622    text: &str,
10623) -> Option<String> {
10624    (usable && !has_unconfirmed_command(commands))
10625        .then(|| verdict::verified_noop(text))
10626        .flatten()
10627}
10628
10629fn short(commit: &str) -> String {
10630    commit.chars().take(7).collect()
10631}
10632
10633fn make_executable(path: &Path) -> Result<()> {
10634    #[cfg(unix)]
10635    {
10636        use std::os::unix::fs::PermissionsExt as _;
10637        let mut perms = std::fs::metadata(path)?.permissions();
10638        perms.set_mode(0o755);
10639        std::fs::set_permissions(path, perms)?;
10640    }
10641    #[cfg(not(unix))]
10642    {
10643        let _ = path;
10644    }
10645    Ok(())
10646}
10647
10648/// What every seat in one batch shares: where the answers are attributed, the
10649/// prompt overlay they inherit, and the build cache they are told to use.
10650///
10651/// A struct rather than four more parameters: `wave` also needs the run's
10652/// state (to record who is answering right now) and the attempt number, and
10653/// eight positional arguments is both unreadable and a clippy error.
10654struct WaveCtx<'a> {
10655    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10656    /// paid for it.
10657    run: &'a str,
10658    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10659    node: &'a str,
10660    prompts: &'a Prompts,
10661    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10662    cache: Option<&'a Path>,
10663    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10664    /// `JobRecord::round`. `None` for every other node.
10665    round: Option<usize>,
10666    /// Carry each seat's failed-agent history across waves (the review loop
10667    /// only): start-of-round priority and handover choice read
10668    /// [`RunState::seat_history`], and every answer or failure writes it.
10669    carry_seats: bool,
10670    /// The run's park / interrupt requests, so a wait for a shared resource
10671    /// can be abandoned the moment the operator asks to park.
10672    pauses: Option<(&'a Pause, &'a Pause)>,
10673}
10674
10675/// Shared scheduling policy for operations and spawned seat invocations.
10676fn repository_wait(
10677    state: &RunState,
10678    pauses: Option<(&Pause, &Pause)>,
10679) -> crate::repo_lock::WaitPolicy {
10680    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10681    let inherited = crate::repo_lock::WaitPolicy::current();
10682    crate::repo_lock::WaitPolicy::new(
10683        Duration::from_secs(state.config.graph.timeout_resource_wait),
10684        move || {
10685            inherited.cancelled()
10686                || pauses
10687                    .as_ref()
10688                    .is_some_and(|(a, b)| a.parked() || b.parked())
10689        },
10690    )
10691}
10692
10693fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10694    if error.is::<crate::repo_lock::ResourceBlocked>() {
10695        state.resource_block = Some(crate::run::ResourceBlock {
10696            cause: crate::run::ResourceBlockCause::Other,
10697            node: "repository".into(),
10698            kind: if error.to_string().contains("disk pressure") {
10699                "disk pressure"
10700            } else if error.to_string().contains("disk admission") {
10701                "disk admission"
10702            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10703                "incremental cache admission"
10704            } else {
10705                "repository coordination"
10706            }
10707            .into(),
10708            detail: error.to_string(),
10709            at: Timestamp::now(),
10710        });
10711        state.event("resource", error.to_string());
10712    }
10713}
10714
10715async fn coordinate_operation(
10716    state: &mut RunState,
10717    cwd: &Path,
10718    wait: &crate::repo_lock::WaitPolicy,
10719) -> Result<Option<crate::repo_lock::SeatLease>> {
10720    let result =
10721        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10722            .await;
10723    if let Err(e) = &result {
10724        record_repository_block(state, e);
10725    }
10726    result
10727}
10728
10729/// Run one job, honouring the parallelism budget.
10730async fn run_one(
10731    job: SeatJob,
10732    sem: Arc<Semaphore>,
10733    ctx: &WaveCtx<'_>,
10734    state: &mut RunState,
10735    attempt: usize,
10736) -> (SeatState, AgentOutcome) {
10737    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10738        .await
10739        .pop()
10740        .expect("one job in, one result out");
10741    (seat, out)
10742}
10743
10744/// Run every job concurrently, capped by the semaphore, preserving order.
10745///
10746/// Every seat in the batch is recorded into [`RunState::active`] before the
10747/// wave starts and cleared as each answer lands, so the run's own record says
10748/// who is still being waited on rather than only who finished.
10749async fn wave(
10750    jobs: Vec<SeatJob>,
10751    sem: Arc<Semaphore>,
10752    ctx: &WaveCtx<'_>,
10753    state: &mut RunState,
10754    attempt: usize,
10755) -> Vec<(usize, SeatState, AgentOutcome)> {
10756    let WaveCtx {
10757        run,
10758        node,
10759        prompts,
10760        cache,
10761        round,
10762        carry_seats: _,
10763        pauses,
10764    } = *ctx;
10765    for job in &jobs {
10766        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10767    }
10768    if let Err(e) = state.save() {
10769        // A failed persist of "who is answering right now" must not abort the
10770        // wave: the seats are already being asked, and the alternative is
10771        // losing the answers to save a status line nobody may even be
10772        // watching.
10773        tracing::warn!("could not persist in-progress seats: {e:#}");
10774    }
10775    // Hold the shared build cache's lease for the whole batch, not per job:
10776    // several candidates (an implement wave) or a fixer legitimately share
10777    // one cache concurrently within this run, and that stays untouched — a
10778    // single lease taken once for the whole wave and released once it is
10779    // done is what stops a *different* borrower (another run's own wave, its
10780    // e2e/gate, a human's `magi review`) from interleaving a build into the
10781    // same directory while this one is in flight. Best-effort, not
10782    // all-or-nothing: a wave that cannot get the lease within its own
10783    // longest job's budget still runs — an hour of paid implementer calls is
10784    // not thrown away over cache contention — but every write-allowed seat
10785    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10786    // below), the same fallback a read-only seat always gets, rather than
10787    // building into a directory this run was never granted. The identity
10788    // record is still invalidated below either way, so the next tracked
10789    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10790    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10791    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10792    // never borrowed from a job's agent budget (and never charged to it either,
10793    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10794    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10795    let cache_guard = if let Some(cache_dir) = cache {
10796        if jobs_had_a_writer {
10797            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10798            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10799            match acquire_cache_lease(
10800                state,
10801                cache_dir,
10802                &owner,
10803                resource_limit,
10804                node,
10805                node,
10806                &cancel,
10807            )
10808            .await
10809            {
10810                Ok(Some(g)) => Some(g),
10811                // Parking: leave without spawning anything. The run parks at
10812                // the next boundary and the seats are asked afresh on resume.
10813                Ok(None) => {
10814                    if let Some(block) = &mut state.resource_block {
10815                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10816                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10817                        };
10818                    }
10819                    return block_wave(
10820                        jobs,
10821                        state,
10822                        node,
10823                        attempt,
10824                        "parked while waiting for the shared build cache",
10825                    );
10826                }
10827                // An unreadable lease is not contention: keep the old
10828                // fallback (no shared cache for this wave's seats).
10829                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10830                Err(e) => {
10831                    // Writing seats may not build into a cache this run does
10832                    // not hold, and an agent that cannot be given one is not
10833                    // asked at all: the run stops resumably instead.
10834                    let why = format!("{e:#}");
10835                    state.event(
10836                        "resource",
10837                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10838                    );
10839                    return block_wave(jobs, state, node, attempt, &why);
10840                }
10841            }
10842        } else {
10843            None
10844        }
10845    } else {
10846        None
10847    };
10848    // Every job keeps its whole budget: the wait above was not the agent's.
10849    // Restart the seats' clocks so "started" means "being spawned now".
10850    state.seats_spawning(node, attempt);
10851    if let Err(e) = state.save() {
10852        tracing::warn!("could not persist the start of a wave: {e:#}");
10853    }
10854    // Queueing behind this wave's own seats for a parallelism slot is normal
10855    // (each of them is bounded by its own agent timeout); only a wait beyond
10856    // all of them plus the resource limit means the slots never came back.
10857    let slot_limit = resource_limit
10858        + jobs
10859            .iter()
10860            .map(|j| j.timeout)
10861            .max()
10862            .unwrap_or_default()
10863            .saturating_mul(jobs.len() as u32);
10864    let mut set = tokio::task::JoinSet::new();
10865    // Seats announce themselves here the moment they hold a parallelism
10866    // permit, i.e. just before their CLI is spawned.
10867    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10868    let overlay = prompts.overlay(node);
10869    let repository_wait = repository_wait(state, pauses);
10870    for (i, mut job) in jobs.into_iter().enumerate() {
10871        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10872        if cache.is_some() {
10873            job.prompt.push('\n');
10874            job.prompt
10875                .push_str(&prompt::build_cache_note(node, job.allow_write));
10876        }
10877        let sem = Arc::clone(&sem);
10878        let run = run.to_owned();
10879        let node = node.to_owned();
10880        // Only implementers were told about the task's attachments, so only
10881        // their seats get the directory widened for reading.
10882        let attachments = if node == "implement" {
10883            state.attachments.clone()
10884        } else {
10885            Vec::new()
10886        };
10887        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10888        // `prompt::build_cache_note`'s doc for why setting it anyway is
10889        // exactly how a sandboxed reviewer's write refusal got reported as a
10890        // defect in the patch, not a property of its own seat. And a
10891        // write-allowed one is handed it only when the lease above was
10892        // actually acquired: a wave that could not get it (`cache_guard` is
10893        // `None`, see its own comment) must not send seats to build into a
10894        // directory this run does not hold - that is the exact concurrent,
10895        // unmanaged-write race this module exists to prevent, not something
10896        // "proceeding anyway" is allowed to reintroduce.
10897        let cache = cache
10898            .filter(|_| job.allow_write && cache_guard.is_some())
10899            .map(Path::to_path_buf);
10900        let spawned_tx = spawned_tx.clone();
10901        let repository_wait = repository_wait.clone();
10902        let coordination = state.config.coordination.clone();
10903        let fetched_base = state.fetched_base.clone();
10904        set.spawn(async move {
10905            // A wait for a parallelism slot is scheduling, not the agent: it
10906            // is bounded by its own limit and ends as a resource block, never
10907            // as a timeout of the agent that was not started.
10908            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10909                Ok(p) => p,
10910                Err(_) => {
10911                    return (
10912                        i,
10913                        job.seat,
10914                        AgentOutcome::ResourceBlocked(format!(
10915                            "no parallel slot became free within {}s",
10916                            slot_limit.as_secs()
10917                        )),
10918                        false,
10919                    );
10920                }
10921            };
10922            spawned_tx.send(job.seat.key.clone()).ok();
10923            let mut seat = job.seat;
10924            let out = repository_wait
10925                .scope(agent::invoke(
10926                    &job.spec,
10927                    &mut seat,
10928                    &Invocation {
10929                        coordination: coordination.as_ref(),
10930                        fetched_base: fetched_base.as_ref(),
10931                        cwd: &job.cwd,
10932                        prompt: &job.prompt,
10933                        timeout: job.timeout,
10934                        allow_write: job.allow_write,
10935                        sessions: job.sessions,
10936                        artifacts: &job.artifacts,
10937                        stem: &job.stem,
10938                        run: &run,
10939                        node: &node,
10940                        cache_dir: cache.as_deref(),
10941                        attachments: &attachments,
10942                        writable: &[],
10943                    },
10944                ))
10945                .await;
10946            let interrupted = out
10947                .as_ref()
10948                .err()
10949                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10950            let out = match out {
10951                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10952                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10953                // Billed work the CLI failed to hand over is not an ordinary
10954                // failure, but its text is the CLI's raw error JSON, not an
10955                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10956                // read it as one by forgetting to check. `usable()` is always
10957                // false here (dropped implies an empty response), so this has
10958                // to be checked before the catch-all `Failed` below or the
10959                // one shape this exists for is lost with the rest.
10960                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10961                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10962                Ok(o) => AgentOutcome::Failed(format!(
10963                    "process exited with no usable output ({})",
10964                    crate::notices::exit_status(o.exit_code)
10965                )),
10966                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10967                    AgentOutcome::ResourceBlocked(e.to_string())
10968                }
10969                Err(e) => AgentOutcome::Failed(e.to_string()),
10970            };
10971            (i, seat, out, interrupted)
10972        });
10973    }
10974    drop(spawned_tx);
10975    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10976    loop {
10977        let joined = tokio::select! {
10978            Some(key) = spawned_rx.recv() => {
10979                // The clock of this seat starts now, not when it queued.
10980                state.seat_spawned(&key);
10981                if let Err(e) = state.save() {
10982                    tracing::warn!("could not persist a seat's start: {e:#}");
10983                }
10984                continue;
10985            }
10986            joined = set.join_next() => joined,
10987        };
10988        let Some(joined) = joined else {
10989            break;
10990        };
10991        let (i, seat, out, interrupted) = match joined {
10992            Ok(v) => v,
10993            // No seat to clear: a panicked task never reported which one it
10994            // was. The defensive sweep below this loop is what stops that
10995            // seat's `active` entry from surviving forever.
10996            Err(e) => {
10997                tracing::error!("agent task panicked: {e}");
10998                continue;
10999            }
11000        };
11001        state.seat_finished(&seat.key);
11002        if interrupted
11003            && node == "implement"
11004            && let Some(label) = state
11005                .candidates
11006                .iter()
11007                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11008                .map(|candidate| candidate.label)
11009        {
11010            state.candidate_preservation.insert(
11011                label,
11012                "interrupted implementation awaiting persistence".into(),
11013            );
11014            state.interrupted_implementations.insert(label);
11015        }
11016        if let AgentOutcome::ResourceBlocked(why) = &out
11017            && state.resource_block.is_none()
11018        {
11019            state.resource_block = Some(crate::run::ResourceBlock {
11020                cause: crate::run::ResourceBlockCause::Other,
11021                node: node.to_owned(),
11022                kind: if why.contains("disk pressure") {
11023                    "disk pressure"
11024                } else if why.contains("disk admission") {
11025                    "disk admission"
11026                } else if why.contains(crate::incremental::BLOCKED) {
11027                    "incremental cache admission"
11028                } else if why.contains("repository coordination") {
11029                    "repository coordination"
11030                } else {
11031                    "parallel slot"
11032                }
11033                .to_owned(),
11034                detail: why.clone(),
11035                at: Timestamp::now(),
11036            });
11037            let activity = if interrupted {
11038                "agent execution interrupted"
11039            } else {
11040                "no agent was started"
11041            };
11042            state.event("resource", format!("{node}: {activity} - {why}"));
11043        }
11044        record_jobs(state, node, round, &seat.key, &out);
11045        if let Err(e) = state.save() {
11046            tracing::warn!("could not persist a seat's completion: {e:#}");
11047        }
11048        if collected.len() <= i {
11049            collected.resize_with(i + 1, || None);
11050        }
11051        collected[i] = Some((i, seat, out));
11052    }
11053    // Belt-and-braces for the panic branch above: every seat this exact batch
11054    // started shares this `(node, attempt)` pair, and every seat that finished
11055    // normally already cleared itself, so anything left tagged with it here
11056    // can only be a panicked task's leftover. Cleared unconditionally rather
11057    // than left to read as still answering forever.
11058    if state
11059        .active
11060        .values()
11061        .any(|a| a.node == node && a.attempt == attempt)
11062    {
11063        state
11064            .active
11065            .retain(|_, a| !(a.node == node && a.attempt == attempt));
11066        if let Err(e) = state.save() {
11067            tracing::warn!("could not persist the end of a wave: {e:#}");
11068        }
11069    }
11070    // Whether or not the lease above was actually held, several worktrees
11071    // may just have built into the cache with nothing here able to name one
11072    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
11073    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
11074    // clean it might not have strictly needed; trusting a stale match would
11075    // cost it a wrong answer.
11076    if let Some(cache_dir) = cache
11077        && jobs_had_a_writer
11078    {
11079        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11080    }
11081    if let Some(guard) = cache_guard {
11082        guard.release();
11083    }
11084    collected.into_iter().flatten().collect()
11085}
11086
11087/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
11088/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
11089/// this is data collection, not the fix-specific completion contract in
11090/// [`Runner::continue_fix_report`], and applies regardless of which node
11091/// asked.
11092///
11093/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11094/// evidence from; `Failed` does not, and correctly contributes nothing — a
11095/// timeout or crash is not itself evidence about a command the seat may have
11096/// started.
11097fn record_jobs(
11098    state: &mut RunState,
11099    node: &str,
11100    round: Option<usize>,
11101    seat: &str,
11102    out: &AgentOutcome,
11103) {
11104    let commands: &[agent::CommandEvidence] = match out {
11105        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11106        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11107    };
11108    let checked_at = Timestamp::now();
11109    for c in commands {
11110        state.jobs.push(JobRecord {
11111            node: node.to_owned(),
11112            round,
11113            seat: seat.to_owned(),
11114            id: c.id.clone(),
11115            description: c.description.clone(),
11116            checked_at,
11117            status: match c.exit_code {
11118                Some(0) => JobStatus::Completed,
11119                Some(_) => JobStatus::Failed,
11120                None => JobStatus::Unknown,
11121            },
11122            exit_code: c.exit_code,
11123            result_summary: c.result_summary.clone(),
11124            source: c.source.clone(),
11125        });
11126    }
11127}
11128
11129/// Is a review round clean, given how many reviewer seats answered against
11130/// how many the round expected?
11131///
11132/// A seat that never answered (timeout, crash, unparsable output) is not a
11133/// seat that read the patch and found nothing — treating it as such is
11134/// exactly the bug this function exists to close. Under the default `block`
11135/// policy a missing seat can never be clean; `warn` still requires the seats
11136/// that *did* answer to have found nothing blocking and verification to be
11137/// green.
11138///
11139/// `quota_missing` narrows that `block` default for exactly one cause of
11140/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11141/// a session limit lifts by the very next round buys nothing — the seat is
11142/// asked again with the same quota — so once every missing seat is accounted
11143/// for by a quota loss (and at least one seat *did* answer, so a decision has
11144/// something to rest on) the round is decided on the panel that could answer,
11145/// same as `warn` would. A panel that lost every seat to quota is not
11146/// decided here: `answered == 0` falls through to the existing `block`
11147/// fallback so a fully collapsed panel still waits rather than landing on no
11148/// review at all.
11149fn round_is_clean(
11150    blocking: usize,
11151    e2e_ok: bool,
11152    answered: usize,
11153    expected: usize,
11154    quota_missing: usize,
11155    policy: IncompleteReviewPolicy,
11156) -> bool {
11157    if blocking != 0 || !e2e_ok {
11158        return false;
11159    }
11160    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11161        return true;
11162    }
11163    answered > 0 && expected - answered <= quota_missing
11164}
11165
11166/// The review loop's own conclusion, derived entirely from its persisted
11167/// round records and the round budget that produced them — never from
11168/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11169/// recomputes the identical answer regardless of what an earlier node in the
11170/// same walk, or a previous walk, did to `status`.
11171///
11172/// `None` while more rounds remain to try, including when review never ran
11173/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11174/// gone clean, or the budget is spent, or the tree has stopped moving (see
11175/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11176///
11177/// - An incomplete panel that raised nothing is missing input, not a
11178///   verified tree — never a hand-off candidate, whatever verification said
11179///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11180/// - Otherwise, green e2e on the last round hands off (see
11181///   [`Runner::stop_reviewing`]); red e2e blocks.
11182///
11183/// A last round whose own verification is still `ResourceBlocked` — magi
11184/// itself never got a command to run, not evidence the patch is broken —
11185/// is neither: this returns `None` for it too, the same as "more rounds
11186/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11187/// handling of that shape) instead of this cheap recomputation guessing a
11188/// verdict a real attempt never produced.
11189pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11190    if max_rounds == 0
11191        || reviews.last().is_some_and(|r| {
11192            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11193        })
11194    {
11195        return Some(RunStatus::Gating);
11196    }
11197    let last = reviews.last()?;
11198    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11199    if reviews.len() < max_rounds && !stagnant {
11200        return None;
11201    }
11202    if last.incomplete() && last.blocking == 0 {
11203        return Some(RunStatus::Blocked);
11204    }
11205    if last.e2e_status() == E2eStatus::ResourceBlocked
11206        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11207    {
11208        return None;
11209    }
11210    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11211        RunStatus::Gating
11212    } else {
11213        RunStatus::Blocked
11214    })
11215}
11216
11217/// How long a re-ask may take, given the budget the first attempt had.
11218///
11219/// A `nudged` retry is a request to restate an answer the seat has already
11220/// worked out: it carries no new work, so it does not deserve the original
11221/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11222/// 133 seconds while a third sat for over ten minutes on a resumed session
11223/// holding 410 KB of prior output - and because the retry had inherited the
11224/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11225/// judging round whose other seats were long finished.
11226///
11227/// A quarter of the budget, with a floor so that a deliberately short timeout
11228/// does not collapse to nothing. A retry that re-sends the whole prompt
11229/// (because the seat kept no context) is the original job again, and keeps the
11230/// original budget.
11231pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11232    if nudged {
11233        (full / 4).max(Duration::from_secs(120)).min(full)
11234    } else {
11235        full
11236    }
11237}
11238
11239/// Run a wave and parse each reply, re-asking the seats whose reply was
11240/// unusable.
11241///
11242/// The re-ask is a nudge rather than the whole prompt again when the seat still
11243/// holds its conversation, which is the difference between a cheap retry and
11244/// paying for the entire candidate set twice.
11245///
11246/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11247/// successor in `roster` is handed to it instead of being re-asked: the
11248/// handover is the retry. A seat with no successor left (a single-agent
11249/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11250/// the asks to one seat in one node number at most
11251/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11252/// is asked once (a dropped stream is nudged first), and only the last agent
11253/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11254/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11255/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11256/// other failure stops the chain when the previous agent failed the same way
11257/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11258/// [`has_context`] is false and the job's own full prompt and full budget are
11259/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11260/// intermediate ones are not losses) and is returned as a failure like any
11261/// other absent seat — the caller decides whether the panel still has a
11262/// quorum. An empty `roster` disables handover: failures are nudged as they
11263/// always were, and a quota is simply lost. A reply that fails to parse or
11264/// validate is the prompt's doing and is only ever nudged, never handed over.
11265///
11266/// The returned [`SeatState`] names the agent that answered (or tried last).
11267#[allow(clippy::too_many_arguments)]
11268async fn ask_json_wave<T>(
11269    jobs: Vec<SeatJob>,
11270    sem: Arc<Semaphore>,
11271    retries: usize,
11272    roster: &[AgentSpec],
11273    ctx: &WaveCtx<'_>,
11274    losses: &mut Vec<QuotaLoss>,
11275    state: &mut RunState,
11276    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11277) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11278where
11279    T: serde::de::DeserializeOwned + Send + 'static,
11280{
11281    ask_wave_with(
11282        jobs,
11283        sem,
11284        retries,
11285        roster,
11286        ctx,
11287        losses,
11288        state,
11289        &|text: &str| {
11290            let v = verdict::extract_json::<T>(text)?;
11291            validate(&v)?;
11292            Ok(v)
11293        },
11294    )
11295    .await
11296}
11297
11298/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11299/// a node whose answer is prose (deliberation) shares the same handover,
11300/// failure classification, quota bookkeeping and bounds instead of a copy.
11301///
11302/// A seat handed to another roster agent is sent the job's `handover` prompt
11303/// (when it has one) rather than `prompt`: the new agent has no session, so a
11304/// resume-style prompt would be incomplete. That holds for the handover ask
11305/// and for every nudge to that agent whose `has_context` is false.
11306#[allow(clippy::too_many_arguments)]
11307async fn ask_wave_with<T>(
11308    jobs: Vec<SeatJob>,
11309    sem: Arc<Semaphore>,
11310    retries: usize,
11311    roster: &[AgentSpec],
11312    ctx: &WaveCtx<'_>,
11313    losses: &mut Vec<QuotaLoss>,
11314    state: &mut RunState,
11315    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11316) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11317where
11318    T: Send + 'static,
11319{
11320    let n = jobs.len();
11321    let originals: Vec<SeatJob> = jobs;
11322    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11323    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11324    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11325    // answer, N once it has gone through N nudges. Read back once this
11326    // returns, so a caller building a history record (`ReviewRecord`) can
11327    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11328    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11329    // field's own doc.
11330    let mut nudges: Vec<usize> = vec![0; n];
11331    // The agent now occupying each seat, the ids it has already been through,
11332    // where in the roster the walk began, the class of the last failure, and
11333    // the stem word of a handover not yet asked (full prompt, full budget).
11334    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11335    let mut tried: Vec<BTreeSet<String>> = specs
11336        .iter()
11337        .map(|s| BTreeSet::from([s.id.clone()]))
11338        .collect();
11339    let starts: Vec<usize> = specs
11340        .iter()
11341        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11342        .collect();
11343    let carry = ctx.carry_seats && !roster.is_empty();
11344    // Carried across rounds: ids that failed the seat earlier, and how the
11345    // last failure went (so a repeat of it is not handed over again).
11346    let carried: Vec<BTreeSet<String>> = originals
11347        .iter()
11348        .map(|j| {
11349            state
11350                .seat_history
11351                .get(&j.seat.key)
11352                .filter(|_| carry)
11353                .map(|h| h.failed.clone())
11354                .unwrap_or_default()
11355        })
11356        .collect();
11357    let mut prev: Vec<Option<FailClass>> = originals
11358        .iter()
11359        .map(|j| {
11360            state
11361                .seat_history
11362                .get(&j.seat.key)
11363                .filter(|_| carry)
11364                .and_then(|h| h.last_fail.clone())
11365        })
11366        .collect();
11367    let next_agent =
11368        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11369            let others: BTreeSet<String> = specs
11370                .iter()
11371                .enumerate()
11372                .filter(|(j, _)| *j != i)
11373                .map(|(_, s)| s.id.clone())
11374                .collect();
11375            pick_successor(
11376                roster,
11377                starts[i],
11378                tried,
11379                carry.then(|| &carried[i]),
11380                &others,
11381            )
11382            .cloned()
11383        };
11384    let mut fresh: Vec<Option<String>> = vec![None; n];
11385    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11386    let mut pending: Vec<usize> = (0..n).collect();
11387
11388    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11389    // asks: an agent with a successor is asked once and handed over, and only
11390    // a seat with no successor spends `retries` nudges on the same agent. This
11391    // only guarantees the loop's own termination whatever those say.
11392    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11393    for round in 0..max_rounds {
11394        if pending.is_empty() {
11395            break;
11396        }
11397        let mut batch = Vec::with_capacity(pending.len());
11398        let mut renudged: Vec<&str> = Vec::new();
11399        for &i in &pending {
11400            let src = &originals[i];
11401            // A seat now held by another agent than the job named has no
11402            // session of its own: it gets the full-context prompt whenever
11403            // it is asked in full (the handover ask, a nudge it cannot
11404            // resume).
11405            let full: &str = match &src.handover {
11406                Some(h) if specs[i].id != src.spec.id => h,
11407                _ => &src.prompt,
11408            };
11409            // The prompt and the budget are one decision: a nudge restates
11410            // finished work, a re-sent prompt redoes it.
11411            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11412                (
11413                    full.to_owned(),
11414                    src.timeout,
11415                    format!("{}-{word}-{}", src.stem, specs[i].id),
11416                )
11417            } else if nudges[i] == 0 {
11418                (full.to_owned(), src.timeout, src.stem.clone())
11419            } else {
11420                renudged.push(src.seat.key.as_str());
11421                let why = done[i]
11422                    .as_ref()
11423                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11424                    .unwrap_or_else(|| "no parsable answer".to_owned());
11425                let nudge = prompt::nudge(&why);
11426                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11427                let prompt = if nudged {
11428                    nudge
11429                } else {
11430                    format!("{full}\n\n---\n\n{nudge}")
11431                };
11432                (
11433                    prompt,
11434                    retry_budget(src.timeout, nudged),
11435                    format!("{}-retry{}", src.stem, nudges[i]),
11436                )
11437            };
11438            batch.push(SeatJob {
11439                spec: specs[i].clone(),
11440                seat: seats[i].clone(),
11441                cwd: src.cwd.clone(),
11442                prompt,
11443                timeout,
11444                allow_write: src.allow_write,
11445                sessions: src.sessions,
11446                artifacts: src.artifacts.clone(),
11447                stem,
11448                handover: None,
11449            });
11450        }
11451
11452        if !renudged.is_empty() {
11453            state.event(
11454                ctx.node,
11455                format!("retry {round}: re-asking {}", renudged.join(", ")),
11456            );
11457        }
11458        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11459        let mut still = Vec::new();
11460        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11461            seats[i] = seat;
11462            // No agent was asked: nothing to hand over, nudge, remember or
11463            // charge. The caller sees `resource_block` and stops the node.
11464            if let AgentOutcome::ResourceBlocked(why) = &out {
11465                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11466                continue;
11467            }
11468            let class = FailClass::of(&out);
11469            // A dropped stream is nudged first (the conversation is still
11470            // there to pick up); only a seat whose nudges are spent hands over.
11471            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11472                && nudges[i] < retries
11473                && !roster.is_empty();
11474            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11475                let next = should_hand_over(prev[i].as_ref(), &cur)
11476                    .then(|| next_agent(i, &tried[i], &specs))
11477                    .flatten();
11478                if carry {
11479                    let h = state
11480                        .seat_history
11481                        .entry(originals[i].seat.key.clone())
11482                        .or_default();
11483                    h.failed.insert(specs[i].id.clone());
11484                    h.last_fail = Some(cur.clone());
11485                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11486                        h.last_ok = None;
11487                    }
11488                    // Saved before the next agent is asked, so a restart in
11489                    // between does not forget who failed.
11490                    if let Err(e) = state.save() {
11491                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11492                    }
11493                }
11494                if let Some(next) = next {
11495                    record_handover(
11496                        state,
11497                        ctx.node,
11498                        &originals[i].seat.key,
11499                        &specs[i].id,
11500                        &next.id,
11501                        &cur,
11502                        &fail_reason(&out),
11503                    );
11504                    tried[i].insert(next.id.clone());
11505                    prev[i] = Some(cur.clone());
11506                    seats[i] =
11507                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11508                    specs[i] = next;
11509                    fresh[i] = Some(cur.stem_word().to_owned());
11510                    nudges[i] = 0;
11511                    done[i] = Some(Err(anyhow::anyhow!(
11512                        "handed over after: {}",
11513                        fail_reason(&out)
11514                    )));
11515                    still.push(i);
11516                    continue;
11517                }
11518            }
11519            if carry
11520                && !nudge_first
11521                && let Some(cur) = class.clone()
11522            {
11523                // The chain ended here (no successor, or a repeated failure).
11524                let h = state
11525                    .seat_history
11526                    .entry(originals[i].seat.key.clone())
11527                    .or_default();
11528                h.failed.insert(specs[i].id.clone());
11529                h.last_fail = Some(cur);
11530            }
11531            let parsed = match out {
11532                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11533                AgentOutcome::Quota(o) => {
11534                    last_quota[i] = o.quota.clone();
11535                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11536                }
11537                // Not a parseable answer: the nudge loop re-asks it, which is
11538                // exactly what a dropped stream needs. Just don't hand its raw
11539                // error JSON to `extract_json`.
11540                AgentOutcome::Dropped(o) => {
11541                    let why = o
11542                        .dropped
11543                        .as_ref()
11544                        .map(|d| d.why.as_str())
11545                        .unwrap_or("the CLI ended the stream without delivering its answer");
11546                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11547                }
11548                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11549                    Err(anyhow::anyhow!(e))
11550                }
11551            };
11552            let quota = class == Some(FailClass::Quota);
11553            let failed = parsed.is_err();
11554            if !failed && matches!(ctx.node, "judge" | "vote") {
11555                // A validated answer restores this seat's quorum presence,
11556                // including on the normal path after an all-seat block.
11557                state
11558                    .quota
11559                    .retain(|loss| loss.seat != originals[i].seat.key);
11560            }
11561            done[i] = Some(parsed);
11562            if carry && !failed {
11563                let h = state
11564                    .seat_history
11565                    .entry(originals[i].seat.key.clone())
11566                    .or_default();
11567                h.failed.remove(&specs[i].id);
11568                h.last_ok = Some(specs[i].id.clone());
11569                h.last_fail = None;
11570            }
11571            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11572            // fail the same way; and never re-ask a seat that already parsed.
11573            // A failed agent that still has a successor is not re-asked
11574            // either: the handover was its remedy and has just been refused
11575            // (the chain stops on a repeated failure class). A seat with no
11576            // successor left (a single-agent roster, the roster's tail, or an
11577            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11578            let agent_failure = class.is_some()
11579                && !nudge_first
11580                && !roster.is_empty()
11581                && next_agent(i, &tried[i], &specs).is_some();
11582            if failed && !quota && !agent_failure && nudges[i] < retries {
11583                nudges[i] += 1;
11584                still.push(i);
11585            }
11586        }
11587        pending = still;
11588    }
11589
11590    // One loss per seat whose chain ended on a quota: the intermediate ones
11591    // were absorbed by a handover and are not losses.
11592    for (i, q) in last_quota.into_iter().enumerate() {
11593        if let Some(quota) = q {
11594            losses.push(QuotaLoss::from_report(
11595                originals[i].seat.key.clone(),
11596                ctx.node,
11597                Some(&quota),
11598            ));
11599        }
11600    }
11601
11602    // A stalled quorum already has its own recovery checkpoint. Refresh
11603    // its missing seats before deciding whether it can resume.
11604    if state.status != RunStatus::Stalled
11605        && ctx.node != "deliberate"
11606        && n > 0
11607        && losses.len() == n
11608        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11609    {
11610        block_machine_outage(state, ctx.node, losses);
11611        state.quota.append(losses);
11612    }
11613
11614    seats
11615        .into_iter()
11616        .zip(done)
11617        .zip(nudges)
11618        .map(|((seat, res), attempts)| {
11619            (
11620                seat,
11621                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11622                attempts,
11623            )
11624        })
11625        .collect()
11626}
11627
11628/// A node with no available agent has no task verdict. Preserve the node
11629/// checkpoint and record why a subsequent explicit resume is needed.
11630pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11631    let mut auth: Vec<_> = losses
11632        .iter()
11633        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11634        .map(|loss| loss.agent.clone())
11635        .collect();
11636    auth.sort();
11637    auth.dedup();
11638    let (kind, detail) = if auth.is_empty() {
11639        (
11640            "agent usage limit",
11641            "All seats reached their agent usage limits; resume after usage is available"
11642                .to_owned(),
11643        )
11644    } else {
11645        (
11646            "agent authentication",
11647            format!(
11648                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11649                auth.join(", ")
11650            ),
11651        )
11652    };
11653    state.resource_block = Some(crate::run::ResourceBlock {
11654        node: node.to_owned(),
11655        kind: kind.to_owned(),
11656        cause: crate::run::ResourceBlockCause::Other,
11657        detail: detail.clone(),
11658        at: Timestamp::now(),
11659    });
11660    state.event(node, detail);
11661}
11662
11663/// Did the wave end without asking any agent (see [`block_wave`])?
11664fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11665    results
11666        .iter()
11667        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11668}
11669
11670/// Settle a wave that never spawned anything: every job answers
11671/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11672/// else about the seat (session, history, quota) is touched.
11673fn block_wave(
11674    jobs: Vec<SeatJob>,
11675    state: &mut RunState,
11676    node: &str,
11677    attempt: usize,
11678    why: &str,
11679) -> Vec<(usize, SeatState, AgentOutcome)> {
11680    state.resource_wait = None;
11681    state
11682        .active
11683        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11684    if let Err(e) = state.save() {
11685        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11686    }
11687    jobs.into_iter()
11688        .enumerate()
11689        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11690        .collect()
11691}
11692
11693/// Acquire the shared build cache's lease, waiting out contention within
11694/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11695/// unbounded wait is never acceptable).
11696///
11697/// A first, non-blocking check happens before ever waiting; if it finds the
11698/// lease busy, that fact is logged as a `verify` event *and* flushed with
11699/// [`RunState::save`] immediately — not only once the wait finally succeeds
11700/// or gives up — so a `magi show` run by a different process while this one
11701/// is still waiting reads a `run.json` that says so, rather than whatever it
11702/// looked like before the wait started. The same applies to the terminal
11703/// failure: logged and saved before this returns `Err`, so a caller that
11704/// could not get the lease at all still leaves a legible record of why.
11705async fn acquire_cache_lease(
11706    state: &mut RunState,
11707    cache_dir: &Path,
11708    owner: &crate::cache::Owner,
11709    budget: Duration,
11710    node: &str,
11711    context: &str,
11712    cancel: &(dyn Fn() -> bool + Sync),
11713) -> Result<Option<crate::cache::Guard>> {
11714    let home = crate::run::home();
11715    let started = Instant::now();
11716    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11717        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11718        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11719        Err(e) => {
11720            state.event(
11721                "verify",
11722                format!("{context}: could not check the shared build cache: {e:#}"),
11723            );
11724            if let Err(e2) = state.save() {
11725                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11726            }
11727            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11728        }
11729    };
11730    state.event(
11731        "verify",
11732        format!(
11733            "{context}: waiting for the shared build cache at {} ({})",
11734            cache_dir.display(),
11735            busy.describe()
11736        ),
11737    );
11738    // Seats of this node are waiting, not running: say so, so the countdown a
11739    // reader sees only ever measures an agent that has been spawned.
11740    state.resource_wait = Some(crate::run::ResourceWait {
11741        node: context.to_owned(),
11742        kind: "shared build cache".to_owned(),
11743        holder: busy.describe(),
11744        since: Timestamp::now(),
11745        limit_secs: budget.as_secs(),
11746    });
11747    if let Err(e) = state.save() {
11748        tracing::warn!("could not persist a cache wait: {e:#}");
11749    }
11750    let remaining = budget.saturating_sub(started.elapsed());
11751    let waited = crate::cache::wait_for_unless(
11752        &home,
11753        cache_dir,
11754        owner,
11755        remaining,
11756        Duration::from_secs(5),
11757        cancel,
11758    )
11759    .await;
11760    state.resource_wait = None;
11761    let cause = match &waited {
11762        Ok(Some(_)) => None,
11763        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11764        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11765            Some(crate::run::ResourceBlockCause::Timeout {
11766                waited_secs: started.elapsed().as_secs(),
11767                limit_secs: budget.as_secs(),
11768            })
11769        }
11770        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11771    };
11772    if let Some(cause) = cause {
11773        let detail = match &waited {
11774            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11775            Err(error) => format!("{error:#}"),
11776        };
11777        state.resource_block = Some(crate::run::ResourceBlock {
11778            cause,
11779            node: node.to_owned(),
11780            kind: "shared build cache".into(),
11781            detail,
11782            at: Timestamp::now(),
11783        });
11784        state.event(
11785            "resource",
11786            state.resource_block.as_ref().unwrap().description(),
11787        );
11788    }
11789    if let Err(error) = state.save() {
11790        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11791    }
11792    waited
11793}
11794
11795/// The lease state could not be read at all (unreadable lease file, a
11796/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11797/// known to be contended, so a wave degrades to running without a shared cache
11798/// instead of stopping on a hold that [`crate::triage`] could never release.
11799#[derive(Debug)]
11800struct CacheCheckError(String);
11801
11802impl std::fmt::Display for CacheCheckError {
11803    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11804        f.write_str(&self.0)
11805    }
11806}
11807
11808impl std::error::Error for CacheCheckError {}
11809
11810/// Keep a specific acquisition outcome when the caller adds command evidence.
11811fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11812    state
11813        .resource_block
11814        .get_or_insert_with(|| crate::run::ResourceBlock {
11815            cause: crate::run::ResourceBlockCause::Other,
11816            node: node.to_owned(),
11817            kind: "shared build cache".into(),
11818            detail: format!("{error:#}"),
11819            at: Timestamp::now(),
11820        });
11821    if let Err(error) = state.save() {
11822        tracing::warn!("could not persist cache resource block: {error:#}");
11823    }
11824}
11825
11826/// Run `body` — a verify command batch — while holding the shared build
11827/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11828/// never interleave with another borrower's build against the same
11829/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11830/// timeout, or a human's own `magi review`. See the `cache` module doc for
11831/// why this matters more than Cargo's own per-target locking covers — two
11832/// *different* worktrees building the same package name/version into one
11833/// cache directory is a staleness bug, not a lock contention one.
11834///
11835/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11836/// agent waves do. Repository scheduling also has its own bounded,
11837/// cancellable resource-wait policy. Contention must not consume the
11838/// verification budget: `body` receives the full `budget` after both leases
11839/// are acquired and the cache is fresh. Without a shared cache, only
11840/// repository coordination is acquired before handing over the full budget.
11841///
11842/// A lease that cannot be acquired within the resource wait is reported as a single
11843/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11844/// verification — the same shape a spawn failure already takes in
11845/// [`run_commands`], so a caller need not special-case it.
11846#[allow(clippy::too_many_arguments)]
11847async fn with_cache_lease<'s, F, Fut>(
11848    state: &'s mut RunState,
11849    cache_dir: Option<&Path>,
11850    node: &str,
11851    seat: &str,
11852    worktree: &Path,
11853    head: &str,
11854    budget: Duration,
11855    context: &str,
11856    wait: &crate::repo_lock::WaitPolicy,
11857    body: F,
11858) -> Result<(Vec<CommandOutcome>, bool)>
11859where
11860    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11861    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11862{
11863    // Match editing waves: cache first, repository second. In particular,
11864    // pre_gate must never hold the repository while waiting for a wave's cache.
11865    let started = Instant::now();
11866    let home = crate::run::home();
11867    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11868    let guard = if let Some(cache_dir) = cache_dir {
11869        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11870        match acquire_cache_lease(
11871            state,
11872            cache_dir,
11873            &owner,
11874            resource_limit,
11875            node,
11876            context,
11877            &|| wait.cancelled(),
11878        )
11879        .await
11880        .and_then(|g| {
11881            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11882        }) {
11883            Ok(g) => Some(g),
11884            Err(e) => {
11885                record_cache_error(state, node, &e);
11886                return Ok((
11887                    vec![CommandOutcome {
11888                        command: "(waiting for the shared build cache)".into(),
11889                        code: None,
11890                        output_tail: e.to_string(),
11891                        duration_ms: started.elapsed().as_millis() as u64,
11892                        resource_blocked: true,
11893                    }],
11894                    false,
11895                ));
11896            }
11897        }
11898    } else {
11899        None
11900    };
11901    let _coordination = coordinate_operation(state, worktree, wait).await?;
11902    if let Some(cache_dir) = cache_dir {
11903        let identity = crate::cache::Identity::new(worktree, head);
11904        if let Err(e) = crate::cache::ensure_fresh(
11905            &home,
11906            cache_dir,
11907            &identity,
11908            &state.config.verify.cargo_manifests(),
11909        ) {
11910            record_cache_error(state, node, &e);
11911            state.event(
11912                "verify",
11913                format!(
11914                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11915                    worktree.display(),
11916                    short(head)
11917                ),
11918            );
11919            if let Some(guard) = guard {
11920                guard.release();
11921            }
11922            return Ok((
11923                vec![CommandOutcome {
11924                    command: "(confirming the shared build cache is fresh)".into(),
11925                    code: None,
11926                    output_tail: e.to_string(),
11927                    duration_ms: started.elapsed().as_millis() as u64,
11928                    resource_blocked: true,
11929                }],
11930                false,
11931            ));
11932        }
11933    }
11934    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11935    if !timed_out_pids.is_empty() {
11936        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11937    }
11938    // Drop repository coordination before releasing the cache, reversing the
11939    // acquisition order. A cancelled/error future also drops both guards.
11940    drop(_coordination);
11941    if let Some(guard) = guard {
11942        guard.release();
11943    }
11944    Ok((outcomes, retried))
11945}
11946
11947/// Poll `pids` — commands [`run_commands`] reports as still running when its
11948/// own timeout elapsed — until every one is confirmed gone, or
11949/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11950///
11951/// Real confirmation where confirmation is possible, not a substitute for
11952/// full process-tree observation: a grandchild the timed-out process spawned
11953/// and that survives independently of it is invisible to a pid check the
11954/// same way it always was, and continuing to observe and collect *that*
11955/// stays a different piece of work with its own owner. This only narrows a
11956/// fixed blind wait into an actual check of the pids this process does know
11957/// about.
11958async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11959    wait_for_pids_with(
11960        pids,
11961        crate::proc::pid_alive,
11962        LEASE_RELEASE_POLL,
11963        LEASE_RELEASE_MAX_WAIT,
11964    )
11965    .await;
11966}
11967
11968/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11969/// interval and ceiling supplied by the caller, so the polling *logic* -
11970/// returns as soon as every pid reports dead, gives up at the ceiling
11971/// otherwise - is testable on millisecond durations without asking the real
11972/// OS about a pid at all.
11973async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11974    pids: &[u32],
11975    alive: F,
11976    poll: Duration,
11977    max_wait: Duration,
11978) {
11979    let deadline = Instant::now() + max_wait;
11980    loop {
11981        if pids.iter().all(|&pid| !alive(pid)) {
11982            return;
11983        }
11984        if Instant::now() >= deadline {
11985            return;
11986        }
11987        tokio::time::sleep(poll).await;
11988    }
11989}
11990
11991/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11992/// admission that it could not even get a verify command to run, as opposed
11993/// to evidence the command actually produced? A caller that would otherwise
11994/// read a resource-blocked outcome as a red command must check this first:
11995/// see [`Runner::gate`], which retries rather than records `Blocked` when
11996/// this is true.
11997fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11998    outcomes.iter().any(|o| o.resource_blocked)
11999}
12000
12001/// What [`Runner::gate_fix_round`] decided.
12002pub(crate) enum GateFix {
12003    /// The tree changed and `verify.e2e` is still green: run the gate again.
12004    Retry,
12005    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
12006    /// last failure stands and the run ends blocked.
12007    Stop,
12008    /// `verify.e2e` could not run after the fix (magi's own contention):
12009    /// decide nothing now, a later reentry retries.
12010    Defer,
12011}
12012
12013/// Is every red command in `outcomes` an ordinary failure the code could
12014/// explain: it ran, exited non-zero, and said something?
12015///
12016/// A timeout, a spawn failure and a killed process all leave `code` `None`;
12017/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
12018/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
12019/// what the command is stays the gate's business.
12020pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12021    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12022    red.peek().is_some()
12023        && red.all(|o| {
12024            !o.resource_blocked
12025                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12026                && !o.output_tail.trim().is_empty()
12027        })
12028}
12029
12030/// Describe one verify command's outcome for the event log, distinguishing a
12031/// build/link failure — the toolchain never produced a binary to run — from
12032/// an actual test failure, since only the latter is a verdict on the patch.
12033fn e2e_outcome_label(o: &CommandOutcome) -> String {
12034    if o.ok() {
12035        return "pass".to_owned();
12036    }
12037    let reason = if o.build_failed() {
12038        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12039    } else {
12040        format!("FAIL ({:?})", o.code)
12041    };
12042    format!(
12043        "{reason}\n{}",
12044        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12045    )
12046}
12047
12048/// Run `verify.e2e`, retrying once if the first attempt could not build or
12049/// link — a build/link failure is frequently a race against a shared
12050/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
12051/// `verify` event per command, tagged with `context` (normally `"round N"`)
12052/// so the two call sites that need this — the ordinary per-round leg in
12053/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
12054/// it will ever call a round green — read identically in the event log.
12055async fn run_e2e_with_retry(
12056    state: &mut RunState,
12057    shell: &[String],
12058    commands: &[String],
12059    worktree: &Path,
12060    timeout: Duration,
12061    context: &str,
12062) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12063    let (mut e2e, mut timed_out_pids) = run_commands(
12064        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12065    )
12066    .await;
12067    for o in &e2e {
12068        state.event(
12069            "verify",
12070            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12071        );
12072    }
12073    // A build/link failure is not a verdict on the patch — it is frequently a
12074    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
12075    // one retry before letting a red like that decide the round.
12076    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12077    if verify_retried {
12078        state.event(
12079            "verify",
12080            format!(
12081                "{context}: verify could not build/link, not a test result — retrying once \
12082                 before concluding"
12083            ),
12084        );
12085        let retried = run_commands(
12086            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12087        )
12088        .await;
12089        e2e = retried.0;
12090        // Both attempts' timeouts matter, not just the last one: the first
12091        // attempt's descendants may still be alive alongside the retry's.
12092        timed_out_pids.extend(retried.1);
12093        for o in &e2e {
12094            state.event(
12095                "verify",
12096                format!(
12097                    "{context}: retry `{}` -> {}",
12098                    o.command,
12099                    e2e_outcome_label(o)
12100                ),
12101            );
12102        }
12103    }
12104    (e2e, verify_retried, timed_out_pids)
12105}
12106
12107/// Run configured shell commands in `cwd`, in order. The second element is
12108/// the pid of every command that hit `timeout` and was still running when
12109/// this stopped waiting on it (best-effort: `None` when the platform did not
12110/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12111/// that releases a shared resource afterward needs to know.
12112///
12113/// Records `task` into [`RunState::active`] at every command boundary
12114/// (`RunState::task_command`) and clears it once the whole list has run
12115/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12116/// for minutes with no seat and no output of its own to show for it (see
12117/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12118/// apart from "not yet run" without this), and this is the only place that
12119/// knows which command is running right now and how many are left. Three
12120/// saves per command — start, not per second — matching the same "only at a
12121/// boundary" rule [`wave`] already follows for seats.
12122#[allow(clippy::too_many_arguments)]
12123async fn run_commands(
12124    state: &mut RunState,
12125    node: &str,
12126    task: &str,
12127    attempt: usize,
12128    shell: &[String],
12129    commands: &[String],
12130    cwd: &Path,
12131    timeout: Duration,
12132) -> (Vec<CommandOutcome>, Vec<u32>) {
12133    if commands.is_empty() {
12134        // Nothing to mark as running and nothing to clear — an empty list
12135        // means "not configured", and touching `active` (or the disk) over
12136        // that would be a write for every round of a repo with no
12137        // `verify.e2e` / `verify.gate` commands at all.
12138        return (Vec::new(), Vec::new());
12139    }
12140    let mut out = Vec::new();
12141    let mut timed_out_pids = Vec::new();
12142    let total = commands.len();
12143    for (idx, command) in commands.iter().enumerate() {
12144        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12145        if let Err(e) = state.save() {
12146            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12147        }
12148        let started = Instant::now();
12149        let mut cmd = tokio::process::Command::new(&shell[0]);
12150        cmd.quiet();
12151        // Verify the checkout as a standalone caller. Worker ownership and
12152        // pinned executable selection belong to this driver, not to tests or
12153        // tools it runs. Preserve home/seat context and Running's containment.
12154        for key in crate::worker::INVOCATION_ENV {
12155            cmd.env_remove(key);
12156        }
12157        cmd.args(&shell[1..])
12158            .arg(command)
12159            .current_dir(cwd)
12160            .stdin(std::process::Stdio::null())
12161            .stdout(std::process::Stdio::piped())
12162            .stderr(std::process::Stdio::piped())
12163            .kill_on_drop(true);
12164        let mut outputs = Vec::new();
12165        let protected = (|| -> Result<()> {
12166            crate::incremental::configure(&mut cmd, cwd)?;
12167            let mut paths = vec![cwd.to_owned()];
12168            if let Some(cache) = state.config.cache_dir() {
12169                paths.push(cache);
12170            }
12171            outputs.extend(crate::retention::OutputLease::admit(
12172                &crate::run::home(),
12173                &paths,
12174                state.config.disk.min_free_bytes,
12175            )?);
12176            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12177                outputs.push(guard);
12178            }
12179            if let Some(cache) = state.config.cache_dir() {
12180                outputs.push(crate::retention::OutputLease::cache(
12181                    &crate::run::home(),
12182                    &cache,
12183                    false,
12184                )?);
12185            }
12186            Ok(())
12187        })();
12188        if let Err(e) = protected {
12189            let detail = format!("execution admission unavailable: {e:#}");
12190            state.resource_block = Some(crate::run::ResourceBlock {
12191                cause: crate::run::ResourceBlockCause::Other,
12192                node: node.to_owned(),
12193                kind: if detail.contains("disk admission") {
12194                    "disk admission"
12195                } else {
12196                    "output ownership"
12197                }
12198                .to_owned(),
12199                detail: detail.clone(),
12200                at: Timestamp::now(),
12201            });
12202            out.push(CommandOutcome {
12203                command: command.clone(),
12204                code: None,
12205                output_tail: detail,
12206                duration_ms: 0,
12207                resource_blocked: true,
12208            });
12209            break;
12210        }
12211        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12212        let (code, body) = match spawned {
12213            Ok(mut running) => {
12214                let pid = running.child().id();
12215                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12216                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12217                let mut paths = vec![cwd.to_owned()];
12218                if let Some(cache) = state.config.cache_dir() {
12219                    paths.push(cache);
12220                }
12221                let mut pressure = None;
12222                let status = tokio::select! {
12223                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12224                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12225                        pressure = Some(reason); None
12226                    }
12227                };
12228                let (code, failure) = match status {
12229                    Some(Ok(Ok(status))) => (
12230                        status.code(),
12231                        status
12232                            .code()
12233                            .is_none()
12234                            .then(|| format!("command terminated: {status}")),
12235                    ),
12236                    Some(Ok(Err(error))) => {
12237                        (None, Some(format!("failed waiting for command: {error}")))
12238                    }
12239                    Some(Err(_)) => {
12240                        // Leased cleanup confirms process-tree death itself.
12241                        // Unleased commands still need the caller's PID check
12242                        // after kill-on-drop, before releasing cache ownership.
12243                        if !running.coordinated()
12244                            && let Some(pid) = pid
12245                        {
12246                            timed_out_pids.push(pid);
12247                        }
12248                        (
12249                            None,
12250                            Some(format!("timed out after {}s", timeout.as_secs())),
12251                        )
12252                    }
12253                    None => (None, None),
12254                };
12255                running.finish().await;
12256                if let Some(reason) = pressure {
12257                    state.resource_block = Some(crate::run::ResourceBlock {
12258                        cause: crate::run::ResourceBlockCause::Other,
12259                        node: node.to_owned(),
12260                        kind: "disk pressure".to_owned(),
12261                        detail: reason.clone(),
12262                        at: Timestamp::now(),
12263                    });
12264                    out.push(CommandOutcome {
12265                        command: command.clone(),
12266                        code: None,
12267                        output_tail: reason,
12268                        duration_ms: started.elapsed().as_millis() as u64,
12269                        resource_blocked: true,
12270                    });
12271                    break;
12272                }
12273                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12274                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12275                if let Some(failure) = failure {
12276                    body.push('\n');
12277                    body.push_str(&failure);
12278                }
12279                (code, body)
12280            }
12281            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12282        };
12283        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12284        if !body.is_empty() {
12285            // The full text is kept whatever the summary dropped. A fresh
12286            // name per command so a resume or retry never overwrites an
12287            // earlier run's output. A failed write changes no verdict.
12288            match crate::run::write_verification_artifact(state, task, &body) {
12289                Ok(name) if summary != body => {
12290                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12291                }
12292                Ok(_) => {}
12293                Err(e) => {
12294                    let warning = format!("could not save the full {task} output: {e:#}");
12295                    tracing::warn!("{warning}");
12296                    state.event(node, warning);
12297                    if summary != body {
12298                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12299                    }
12300                }
12301            }
12302        }
12303        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12304        out.push(CommandOutcome {
12305            command: command.clone(),
12306            code,
12307            output_tail: summary,
12308            duration_ms: started.elapsed().as_millis() as u64,
12309            resource_blocked,
12310        });
12311        if resource_blocked {
12312            state.resource_block = Some(crate::run::ResourceBlock {
12313                cause: crate::run::ResourceBlockCause::Other,
12314                node: node.to_owned(),
12315                kind: "incremental cache admission".into(),
12316                detail: body,
12317                at: Timestamp::now(),
12318            });
12319            break;
12320        }
12321    }
12322    state.task_finished(task);
12323    if let Err(e) = state.save() {
12324        tracing::warn!("could not persist the end of {task}: {e:#}");
12325    }
12326    (out, timed_out_pids)
12327}
12328
12329/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12330/// section (`report::run`) and in the `merge` event this node records — for
12331/// the operator to run by hand.
12332///
12333/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12334/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12335/// merge commits", or "require linear history") rejects the push a `--no-ff`
12336/// merge would produce, which is exactly the guidance this function replaces.
12337/// `message`'s first line becomes the squash commit's subject, matching the
12338/// note `report::run` prints alongside this command — see that function for
12339/// why an explicit subject is not optional there.
12340fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12341    let repo = repo.display();
12342    match style {
12343        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12344        MergeStyle::Squash => {
12345            // The subject sits inside double quotes, and a title an agent
12346            // wrote may carry the characters that break out of them.
12347            let subject = message
12348                .lines()
12349                .next()
12350                .unwrap_or(branch)
12351                .replace(['\\', '"', '$', '`'], "");
12352            format!(
12353                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12354            )
12355        }
12356        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12357    }
12358}
12359
12360/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12361/// under the hood, rejects a `title` over 256 characters and the whole
12362/// command fails — no PR at all, for a run whose body was otherwise fine
12363/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12364/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12365/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12366/// this bound. It is a margin, not a guarantee — a title packed
12367/// with multi-unit characters could still in principle land close to the
12368/// edge, but a real task title's occasional emoji or accented letter fits
12369/// comfortably inside it.
12370const PR_TITLE_MAX: usize = 240;
12371
12372/// A pull request title from the opening line of `text`, or `None` when that
12373/// line is not English (GitHub text is) or has no letters.
12374///
12375/// A line within `max` is kept as is. A longer one is cut at the end of its
12376/// first sentence when that falls inside `max`, else at a word boundary with a
12377/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12378/// make every merely-truncated title look non-English). The language check
12379/// runs on the kept text before any mark is added, so only what GitHub will
12380/// show is judged: an English opening followed by non-ASCII far past the cut
12381/// still passes.
12382fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12383    let line = queue::first_line(text)?;
12384    let chars: Vec<char> = line.chars().collect();
12385    let (kept, mark) = if chars.len() <= max {
12386        (line.to_owned(), "")
12387    } else if let Some(end) = sentence_end(&chars, max) {
12388        (chars[..end].iter().collect::<String>(), "")
12389    } else {
12390        let room = max.saturating_sub(3);
12391        // Cut at the last space inside the room; when the char just past the
12392        // room is a space the room already ends on a word.
12393        let cut = if chars[room].is_whitespace() {
12394            room
12395        } else {
12396            chars[..room]
12397                .iter()
12398                .rposition(|c| c.is_whitespace())
12399                .unwrap_or(room)
12400        };
12401        let head: String = chars[..cut].iter().collect();
12402        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12403        (head.to_owned(), "...")
12404    };
12405    if kept.is_empty() {
12406        return None;
12407    }
12408    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12409        return None;
12410    }
12411    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12412        return None;
12413    }
12414    Some(format!("{kept}{mark}"))
12415}
12416
12417/// The char length of the first sentence of `chars` when it ends within `max`
12418/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12419/// abbreviations so `e.g. foo` does not end a title early.
12420fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12421    const MIN: usize = 20;
12422    for i in MIN..max.min(chars.len()) {
12423        if !matches!(chars[i], '.' | '!' | '?') {
12424            continue;
12425        }
12426        let Some(&next) = chars.get(i + 1) else {
12427            continue;
12428        };
12429        if !next.is_whitespace() {
12430            continue;
12431        }
12432        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12433        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12434            continue;
12435        }
12436        let word: String = chars[..i]
12437            .iter()
12438            .rev()
12439            .take_while(|c| !c.is_whitespace())
12440            .collect::<Vec<_>>()
12441            .into_iter()
12442            .rev()
12443            .collect();
12444        let word = word.to_ascii_lowercase();
12445        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12446            continue;
12447        }
12448        let end = chars[..i]
12449            .iter()
12450            .rposition(|c| !c.is_whitespace())
12451            .map_or(i, |p| p + 1);
12452        return Some(end);
12453    }
12454    None
12455}
12456
12457/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12458/// change: a title and a body describing what was *implemented*, not the task
12459/// that asked for it. A task reads as a request; a reader of the merged
12460/// history wants the change.
12461struct PrMessage {
12462    title: String,
12463    body: String,
12464}
12465
12466impl PrMessage {
12467    /// Title, blank line, body. The first line is the squash/merge commit
12468    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12469    /// has to stay one sensible line.
12470    fn commit_message(&self) -> String {
12471        format!("{}\n\n{}", self.title, self.body)
12472    }
12473}
12474
12475/// The text after a leading `TITLE:` (any case) on `line`.
12476fn title_marker(line: &str) -> Option<&str> {
12477    let line = line.trim();
12478    let head = line.get(..6)?;
12479    head.eq_ignore_ascii_case("title:")
12480        .then(|| line[6..].trim())
12481}
12482
12483/// The implementer's own one-line title: the `TITLE:` line the implement
12484/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12485/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12486/// source, and a title that says as much is refused here too.
12487fn summary_title(summary: &str) -> Option<String> {
12488    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12489    let raw = title_marker(first)?;
12490    if raw.is_empty() {
12491        return None;
12492    }
12493    let title = queue::title_from(raw, PR_TITLE_MAX);
12494    let lower = title.to_ascii_lowercase();
12495    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12496        return None;
12497    }
12498    Some(title)
12499}
12500
12501/// How `open_review`'s instruction begins; see [`landing_title`].
12502const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12503
12504/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12505pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12506
12507/// Does this run review work that already existed, rather than implement a
12508/// task? Runs recorded before `reviewed_commits` existed carry only the
12509/// candidate marker.
12510pub(crate) fn is_review_run(state: &RunState) -> bool {
12511    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12512}
12513
12514/// The title of a review-only run: the subject of the oldest commit under
12515/// review. Later commits are usually fixups, and `instruction` is the review
12516/// prompt, which says nothing about the change. GitHub text is English, so a
12517/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12518fn review_title(state: &RunState) -> Option<String> {
12519    english_subject(
12520        state.reviewed_commits.as_ref()?.first()?,
12521        state.config.graph.github_english(),
12522    )
12523}
12524
12525/// `raw` as a pull request title, or `None` when it is blank, not English
12526/// (GitHub text is), or one of magi's own candidate commit subjects.
12527fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12528    let raw = raw.trim();
12529    if raw.is_empty() {
12530        return None;
12531    }
12532    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12533        return None;
12534    }
12535    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12536        return None;
12537    }
12538    let title = queue::title_from(raw, PR_TITLE_MAX);
12539    let lower = title.to_ascii_lowercase();
12540    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12541        return None;
12542    }
12543    Some(title)
12544}
12545
12546/// What a review-only run's branch says about itself, read at the moment the
12547/// pull request is opened.
12548#[derive(Debug, Clone, PartialEq, Eq)]
12549struct BranchFacts {
12550    /// `(subject, body)` of each commit, oldest first.
12551    commits: Vec<(String, String)>,
12552    /// Trimmed `git diff --stat`.
12553    stat: String,
12554}
12555
12556/// Longest diff stat shown: this many file lines plus the summary line.
12557const STAT_FILE_LINES: usize = 25;
12558/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12559const COMMITS_MAX_CHARS: usize = 20_000;
12560
12561/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12562/// say or finds nothing, so the caller falls back to what the run recorded.
12563async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12564    let commits = git::commit_log(repo, base, branch).await.ok()?;
12565    if commits.is_empty() {
12566        return None;
12567    }
12568    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12569    let lines: Vec<&str> = stat.lines().collect();
12570    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12571        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12572        let more = format!(" ... {omitted} more file(s)");
12573        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12574        kept.push(&more);
12575        kept.push(lines[lines.len() - 1]);
12576        kept.join("\n")
12577    } else {
12578        lines.join("\n")
12579    };
12580    Some(BranchFacts { commits, stat })
12581}
12582
12583/// Defang what would break the surrounding markdown: a closing `</details>`
12584/// and a code fence.
12585fn markdown_safe(text: &str) -> String {
12586    text.replace("</details>", "&lt;/details&gt;")
12587        .replace("\x60\x60\x60", "~~~")
12588}
12589
12590fn neutral_title(state: &RunState, winner: char) -> String {
12591    format!(
12592        "chore: land candidate {} of run {}",
12593        winner.to_ascii_uppercase(),
12594        state.id
12595    )
12596}
12597
12598/// Reuse the graph's bounded fixer without starting another graph or driver.
12599/// Copy the working state back even on errors so spent rounds remain visible.
12600pub(crate) async fn repair_landing_gate(
12601    state: &mut RunState,
12602    winner: &Candidate,
12603    outcomes: &[CommandOutcome],
12604) -> Result<GateFix> {
12605    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12606        return Ok(GateFix::Stop);
12607    }
12608    let mut runner = Runner {
12609        roles: state.config.resolve_roles()?,
12610        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12611        state: state.clone(),
12612        pause: Pause::new(),
12613        interrupt: Pause::new(),
12614    };
12615    let result = runner.gate_fix_round(winner, outcomes).await;
12616    *state = runner.state;
12617    result
12618}
12619
12620/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12621/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12622pub(crate) async fn verify_current_head(
12623    state: &mut RunState,
12624    winner: &Candidate,
12625    wait: &crate::repo_lock::WaitPolicy,
12626) -> Result<bool> {
12627    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12628    // Preserve the previous record before replacing it, including on crash recovery.
12629    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12630        state
12631            .verification_history
12632            .push(crate::run::VerificationEvidence {
12633                superseded_by: Some(head.clone()),
12634                archived_at: Some(Timestamp::now()),
12635                fixes_spent: Some(state.gate_fixes.len()),
12636                gate_head: None,
12637                gate_ran: false,
12638                gate: Vec::new(),
12639                e2e: state.head_e2e.take(),
12640            });
12641    }
12642    if let Some(e) = &state.head_e2e
12643        && e.head == head
12644    {
12645        if e.passed {
12646            return Ok(true);
12647        }
12648        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12649            state.status = RunStatus::Blocked;
12650            state.save()?;
12651            return Ok(false);
12652        }
12653    } else if state.reviews.last().is_some_and(|r| {
12654        r.verified_head.as_deref() == Some(head.as_str())
12655            && (r.e2e_status() == E2eStatus::Passed
12656                || (r.e2e_status() == E2eStatus::NotConfigured
12657                    && state.config.verify.e2e.is_empty()))
12658    }) || state
12659        .gate_repair_e2e
12660        .as_ref()
12661        .is_some_and(|e| e.head == head && e.passed)
12662    {
12663        return Ok(true);
12664    }
12665    if let Some(e) = &state.gate_repair_e2e
12666        && e.head == head
12667        && !e.passed
12668        && !e.outcomes.is_empty()
12669        && !verify_inconclusive(&e.outcomes)
12670    {
12671        state.status = RunStatus::Blocked;
12672        state.save()?;
12673        return Ok(false);
12674    }
12675    state.head_e2e = Some(crate::run::GateRepairE2e {
12676        head: head.clone(),
12677        passed: false,
12678        outcomes: Vec::new(),
12679    });
12680    state.save()?;
12681    let commands = state.config.verify.e2e.clone();
12682    let shell = state.config.shell();
12683    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12684    let cache_dir = state.config.cache_dir();
12685    let outcomes = if commands.is_empty() {
12686        Vec::new()
12687    } else {
12688        let (outcomes, _) = with_cache_lease(
12689            state,
12690            cache_dir.as_deref(),
12691            "e2e",
12692            "e2e",
12693            &winner.worktree,
12694            &head,
12695            timeout,
12696            "current head e2e",
12697            wait,
12698            |state, budget| async move {
12699                run_e2e_with_retry(
12700                    state,
12701                    &shell,
12702                    &commands,
12703                    &winner.worktree,
12704                    budget,
12705                    "current head e2e",
12706                )
12707                .await
12708            },
12709        )
12710        .await?;
12711        outcomes
12712    };
12713    let unresolved = verify_inconclusive(&outcomes);
12714    let passed = !unresolved
12715        && outcomes.iter().all(CommandOutcome::ok)
12716        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12717    if !outcomes.is_empty() && !unresolved {
12718        state.record_base_progress(&head);
12719    }
12720    state.head_e2e = Some(crate::run::GateRepairE2e {
12721        head: head.clone(),
12722        passed,
12723        outcomes,
12724    });
12725    if !passed && !unresolved {
12726        state.status = RunStatus::Blocked;
12727        state.event(
12728            "e2e",
12729            format!("verify.e2e failed on head {head}; nothing published"),
12730        );
12731    }
12732    state.save()?;
12733    Ok(passed)
12734}
12735
12736/// Recheck a repair's e2e requirement across failures, resource deferrals and
12737/// process interruptions. A successful result only applies to its exact SHA.
12738pub(crate) async fn check_gate_repair_e2e(
12739    state: &mut RunState,
12740    winner: &Candidate,
12741    wait: &crate::repo_lock::WaitPolicy,
12742) -> Result<GateFix> {
12743    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12744        return Ok(GateFix::Retry);
12745    }
12746    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12747    if state
12748        .gate_repair_e2e
12749        .as_ref()
12750        .is_some_and(|r| r.head == head && r.passed)
12751    {
12752        return Ok(GateFix::Retry);
12753    }
12754    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12755        head: head.clone(),
12756        passed: false,
12757        outcomes: Vec::new(),
12758    });
12759    state.save()?;
12760    let commands = state.config.verify.e2e.clone();
12761    let outcomes = if commands.is_empty() {
12762        Vec::new()
12763    } else {
12764        let shell = state.config.shell();
12765        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12766        let cache_dir = state.config.cache_dir();
12767        let (outcomes, _) = with_cache_lease(
12768            state,
12769            cache_dir.as_deref(),
12770            "e2e",
12771            "e2e",
12772            &winner.worktree,
12773            &head,
12774            timeout,
12775            "gate repair e2e",
12776            wait,
12777            |state, budget| async move {
12778                run_e2e_with_retry(
12779                    state,
12780                    &shell,
12781                    &commands,
12782                    &winner.worktree,
12783                    budget,
12784                    "gate repair e2e",
12785                )
12786                .await
12787            },
12788        )
12789        .await?;
12790        outcomes
12791    };
12792    let deferred = verify_inconclusive(&outcomes);
12793    let passed = outcomes.iter().all(CommandOutcome::ok)
12794        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12795    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12796        head,
12797        passed,
12798        outcomes,
12799    });
12800    state.save()?;
12801    if deferred {
12802        Ok(GateFix::Defer)
12803    } else if !passed {
12804        state.event(
12805            "gate",
12806            "verify.e2e failed after gate repair; nothing published",
12807        );
12808        Ok(GateFix::Stop)
12809    } else {
12810        Ok(GateFix::Retry)
12811    }
12812}
12813
12814/// Run the same configured gate for a landing head and a fresh graph.
12815pub(crate) async fn run_landing_gate(
12816    state: &mut RunState,
12817    winner: &Candidate,
12818    wait: &crate::repo_lock::WaitPolicy,
12819) -> Result<Vec<CommandOutcome>> {
12820    let shell = state.config.shell();
12821    let gate_commands = state.config.verify.gate.clone();
12822    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12823    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12824        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12825        Vec::new()
12826    } else {
12827        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12828        let cache_dir = state.config.cache_dir();
12829        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12830        let (outcomes, _) = with_cache_lease(
12831            state,
12832            cache_dir.as_deref(),
12833            "gate",
12834            "gate",
12835            &winner.worktree,
12836            &head,
12837            timeout,
12838            "final gate",
12839            wait,
12840            |state, budget| {
12841                let shell = shell.clone();
12842                let gate_commands = gate_commands.clone();
12843                let winner = winner.clone();
12844                async move {
12845                    Runner::run_pre_gate(state, &winner).await;
12846                    // Bind command evidence after mechanical fixes, before commands start.
12847                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12848                    let (outcomes, timed_out_pids) = run_commands(
12849                        state,
12850                        "gate",
12851                        "gate",
12852                        0,
12853                        &shell,
12854                        &gate_commands,
12855                        &winner.worktree,
12856                        budget,
12857                    )
12858                    .await;
12859                    (outcomes, false, timed_out_pids)
12860                }
12861            },
12862        )
12863        .await?;
12864        outcomes
12865    };
12866    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12867    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12868        let mut outcomes = outcomes;
12869        outcomes.push(CommandOutcome {
12870            command: "gate head stability".into(), code: None,
12871            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12872            duration_ms: 0, resource_blocked: false,
12873        });
12874        return Ok(outcomes);
12875    }
12876    if outcomes.is_empty() {
12877        // Nothing configured to check — distinct from every other
12878        // silence in this run's event log, since an empty `gate` alone
12879        // no longer says whether the gate ran at all (see
12880        // `RunState::gate_ran`'s own doc).
12881        state.event("gate", "no gate commands configured; not tested");
12882    }
12883    for o in &outcomes {
12884        state.event(
12885            "gate",
12886            format!(
12887                "`{}` -> {}",
12888                o.command,
12889                if o.ok() {
12890                    "pass".to_owned()
12891                } else {
12892                    format!(
12893                        "FAIL ({:?})\n{}",
12894                        o.code,
12895                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12896                    )
12897                }
12898            ),
12899        );
12900    }
12901    Ok(outcomes)
12902}
12903
12904/// The pull request title to hand to `land::merge_subject`. A review-only run
12905/// opened by an earlier build titled its pull request with the review prompt;
12906/// that title is dropped (empty, so the fallback applies) rather than landed.
12907/// Any other title, including an operator's rename, passes through untouched,
12908/// and so does every title of a run that implements a task.
12909pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12910    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12911        ""
12912    } else {
12913        pr_title
12914    }
12915}
12916
12917/// What the squash subject falls back to when the pull request title is empty
12918/// or candidate-shaped: for a review-only run the derived title, never the
12919/// review prompt held in `instruction`.
12920pub fn landing_subject_source(state: &RunState) -> String {
12921    if is_review_run(state) {
12922        let winner = state.candidates.first().map_or('A', |c| c.label);
12923        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12924    }
12925    state.instruction.clone()
12926}
12927
12928/// `summary` without its `TITLE:` line, which the pull request title already
12929/// carries.
12930fn summary_without_title(summary: &str) -> String {
12931    let mut lines = summary.trim().lines().peekable();
12932    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12933        lines.next();
12934    }
12935    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12936}
12937
12938/// The pull request title and body for the winning candidate.
12939///
12940/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12941/// the task's own opening line via [`queue::title_from`] when there is none.
12942/// `state.instruction` can open with blank lines (`task_text` only rejects a
12943/// body that is blank *entirely*), which `title_from` skips.
12944///
12945/// Publishing uses an independent description generated from the final diff
12946/// and structured evidence. Internal summaries and reports remain local.
12947/// Non-publishing previews may display the original task and candidate summary.
12948/// The footer retains the run and candidate provenance.
12949#[cfg(test)]
12950fn pr_message(state: &RunState, winner: char) -> PrMessage {
12951    pr_message_with(state, winner, None)
12952}
12953
12954/// [`pr_message`] with what the branch of a review-only run says about itself.
12955/// `facts` is ignored for a run that implements a task.
12956#[cfg(test)]
12957fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12958    let raw = pr_message_raw(
12959        state,
12960        winner,
12961        facts,
12962        crate::pr_outcome::FinalHead::Unreported,
12963        ChangeNote::NotAsked,
12964    );
12965    let id = crate::scrub::Identity::current();
12966    PrMessage {
12967        title: crate::scrub::scrub(&raw.title, &id),
12968        body: crate::scrub::scrub(&raw.body, &id),
12969    }
12970}
12971
12972/// What the public description may say about the change itself.
12973#[derive(Debug, Clone, Copy)]
12974enum ChangeNote<'a> {
12975    /// A local preview or non-publishing merge mode: no public generation needed.
12976    NotAsked,
12977    /// A summary written from the final diff.
12978    Summary(&'a crate::run::ChangeSummary),
12979    /// A summary was asked for and could not be produced; the text says why.
12980    Unavailable(&'a str),
12981}
12982
12983fn pr_message_raw(
12984    state: &RunState,
12985    winner: char,
12986    facts: Option<&BranchFacts>,
12987    head: crate::pr_outcome::FinalHead<'_>,
12988    change: ChangeNote<'_>,
12989) -> PrMessage {
12990    if !matches!(change, ChangeNote::NotAsked) {
12991        let (title, mut body) = match change {
12992            ChangeNote::Summary(c) => (
12993                c.title.clone(),
12994                format!("## Summary\n\n{}\n\n", c.body.trim()),
12995            ),
12996            ChangeNote::Unavailable(why) => (
12997                neutral_title(state, winner),
12998                format!(
12999                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13000                ),
13001            ),
13002            ChangeNote::NotAsked => unreachable!(),
13003        };
13004        body.push_str(&public_pr_evidence(state, head));
13005        body.push_str(&format!(
13006            "\n---\nmagi:run/{} magi:candidate-{}\n",
13007            state.id,
13008            winner.to_ascii_lowercase()
13009        ));
13010        return PrMessage { title, body };
13011    }
13012    pr_preview(state, winner, facts, head)
13013}
13014
13015/// Append public facts from the run records after every prose generation,
13016/// including posting-gate rewrites. The agent cannot drop this provenance.
13017fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13018    let mut body = crate::pr_outcome::review_rounds(state);
13019    body.push_str(&crate::pr_outcome::final_verification(state, head));
13020    let open = state.last_round_findings();
13021    if !open.is_empty() {
13022        body.push_str("## Open review findings\n\n");
13023        for f in open {
13024            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13025        }
13026        body.push('\n');
13027    }
13028    if let Some(c) = state.change_summary.as_ref()
13029        && head == crate::pr_outcome::FinalHead::At(&c.head)
13030    {
13031        body.push_str(&format!(
13032            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13033            c.base, c.head
13034        ));
13035    }
13036    body
13037}
13038
13039/// Local-only preview of the operator's task and candidate summary.
13040fn pr_preview(
13041    state: &RunState,
13042    winner: char,
13043    facts: Option<&BranchFacts>,
13044    head: crate::pr_outcome::FinalHead<'_>,
13045) -> PrMessage {
13046    let summary = state
13047        .candidates
13048        .iter()
13049        .find(|c| c.label == winner)
13050        .map(|c| c.summary.as_str())
13051        .unwrap_or_default();
13052    // The fallback is the operator's own words and may not be English; GitHub
13053    // text always is, so a non-English task gets a neutral title instead.
13054    let review = is_review_run(state);
13055    let title = if review {
13056        facts
13057            .and_then(|f| {
13058                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13059            })
13060            .or_else(|| review_title(state))
13061            .or_else(|| {
13062                state
13063                    .candidates
13064                    .iter()
13065                    .find(|c| c.label == winner)
13066                    .filter(|c| !c.branch.starts_with("magi/"))
13067                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13068            })
13069            .unwrap_or_else(|| neutral_title(state, winner))
13070    } else {
13071        summary_title(summary).unwrap_or_else(|| {
13072            english_title(
13073                &state.instruction,
13074                PR_TITLE_MAX,
13075                state.config.graph.github_english(),
13076            )
13077            .unwrap_or_else(|| neutral_title(state, winner))
13078        })
13079    };
13080
13081    let mut body = String::new();
13082    let what = summary_without_title(summary);
13083    if !what.is_empty() {
13084        body.push_str("## Summary\n\n");
13085        body.push_str(&what);
13086        body.push_str("\n\n");
13087    }
13088
13089    // Review history uses structured facts across all rounds, never reports.
13090    body.push_str(&crate::pr_outcome::review_rounds(state));
13091    body.push_str(&crate::pr_outcome::final_verification(state, head));
13092    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13093
13094    let open = state.open_findings();
13095    if !open.is_empty() {
13096        body.push_str("## Open review findings\n\n");
13097        for f in &open {
13098            body.push_str(&format!(
13099                "- `{}` [{:?}] {}\n",
13100                f.id, f.severity, "Context retained in local review records."
13101            ));
13102        }
13103        body.push('\n');
13104    }
13105
13106    if let Some(fix) = fix
13107        && !fix.rejected.is_empty()
13108    {
13109        body.push_str("## Declined by the fixer\n\n");
13110        for r in &fix.rejected {
13111            body.push_str(&format!(
13112                "- `{}`: {}\n",
13113                r.id, "Reason retained in local review records."
13114            ));
13115        }
13116        body.push('\n');
13117    }
13118
13119    if review {
13120        // The review prompt is not the task; list what the branch carries.
13121        body.push_str("## Commits under review\n\n");
13122        if let Some(facts) = facts {
13123            let mut left = COMMITS_MAX_CHARS;
13124            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13125                let mut entry = format!("- {}\n", markdown_safe(subject));
13126                for l in markdown_safe(text).lines() {
13127                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13128                }
13129                if left == 0 {
13130                    body.push_str(&format!(
13131                        "- ... {} more commit(s)\n",
13132                        facts.commits.len() - i
13133                    ));
13134                    break;
13135                }
13136                if entry.len() > left {
13137                    // Even the first commit is cut: one huge body must not
13138                    // push the whole description past GitHub's limit.
13139                    let mut end = left;
13140                    while !entry.is_char_boundary(end) {
13141                        end -= 1;
13142                    }
13143                    entry.truncate(end);
13144                    entry.push_str("\n  ... (truncated)\n");
13145                    left = 0;
13146                } else {
13147                    left -= entry.len();
13148                }
13149                body.push_str(&entry);
13150            }
13151            if !facts.stat.trim().is_empty() {
13152                body.push_str(&format!(
13153                    "\n## Diff stat\n\n```\n{}\n```\n",
13154                    markdown_safe(facts.stat.trim())
13155                ));
13156            }
13157        } else {
13158            match &state.reviewed_commits {
13159                Some(subjects) => {
13160                    for s in subjects {
13161                        body.push_str(&format!("- {}\n", s.trim()));
13162                    }
13163                }
13164                None => {
13165                    // An older run kept only the prompt, with the commit list
13166                    // after its first paragraph.
13167                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13168                    body.push_str(rest.trim());
13169                    body.push('\n');
13170                }
13171            }
13172        }
13173    } else {
13174        let task = state.instruction.trim();
13175        let task = if task.is_empty() {
13176            "(empty task)"
13177        } else {
13178            task
13179        };
13180        body.push_str(&format!(
13181            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13182            task.replace("</details>", "&lt;/details&gt;")
13183        ));
13184    }
13185
13186    body.push_str(&format!(
13187        "\n---\nmagi:run/{} magi:candidate-{}\n",
13188        state.id,
13189        winner.to_ascii_lowercase()
13190    ));
13191
13192    PrMessage { title, body }
13193}
13194
13195/// Source imports have already been prepared from structured inputs. Legacy
13196/// inferred references must never instruct a seat to reconstruct those imports.
13197fn seeded_instruction(state: &RunState) -> String {
13198    let mut inputs = Vec::new();
13199    if let Some(start) = &state.sources.start_from {
13200        inputs.push(format!(
13201            "Start from `{}` at `{}` (including ancestry).",
13202            start.token, start.sha
13203        ));
13204    }
13205    for pick in &state.sources.pick_commits {
13206        inputs.push(format!(
13207            "Pick `{}` at `{}` (single commit).",
13208            pick.token, pick.sha
13209        ));
13210    }
13211    if inputs.is_empty() {
13212        state.instruction.clone()
13213    } else {
13214        format!(
13215            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13216            state.instruction,
13217            inputs.join("\n")
13218        )
13219    }
13220}
13221
13222/// Does the winner have no commits ahead of the base it would land on?
13223/// Any failure to find out reads as "not empty": the merge then behaves as it
13224/// always did rather than refusing on a guess.
13225async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13226    let base = &state.base_branch;
13227    let mut against = base.clone();
13228    if mode == MergeMode::Pr {
13229        // A pull request lands on the remote's base, never the local branch
13230        // of the same name: if that cannot be read, "not empty" is the safe
13231        // answer.
13232        let remote = &state.config.merge.remote;
13233        let tracking = format!("{remote}/{base}");
13234        let fetched = git::fetch(repo, remote, base).await;
13235        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13236            against = tracking;
13237        } else {
13238            return false;
13239        }
13240    }
13241    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13242}
13243
13244/// Why nothing was opened for an empty winner, with what the task's own
13245/// references resolved to.
13246fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13247    let mut detail = format!(
13248        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13249         nothing to open a pull request for"
13250    );
13251    detail.push_str(
13252        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13253    );
13254    detail
13255}
13256
13257/// What the `Pr` merge does once it knows whether the branch already has an
13258/// open pull request.
13259#[derive(Debug, PartialEq, Eq)]
13260enum PrPlan {
13261    Create,
13262    Adopt { url: String, title: String },
13263    Stop(String),
13264}
13265
13266/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13267/// a failed lookup -> stop with the real reason. Never guesses.
13268fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13269    match found {
13270        Ok(land::OpenPr::None) => PrPlan::Create,
13271        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13272        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13273            "several open pull requests exist for this branch, not picking one: {}",
13274            urls.join(" ")
13275        )),
13276        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13277    }
13278}
13279
13280/// `gh pr create`, returning the PR url.
13281pub(crate) async fn gh_pr_create(
13282    cwd: &Path,
13283    base: &str,
13284    head: &str,
13285    title: &str,
13286    body: &str,
13287) -> Result<String> {
13288    let out = tokio::process::Command::new("gh")
13289        .args([
13290            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13291        ])
13292        .current_dir(cwd)
13293        .quiet()
13294        .stdin(std::process::Stdio::null())
13295        .output()
13296        .await
13297        .context("spawn gh")?;
13298    if out.status.success() {
13299        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13300    } else {
13301        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13302    }
13303}
13304
13305/// Never force deletion of uncommitted work, including unknown legacy trees.
13306async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13307    match std::fs::symlink_metadata(path) {
13308        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13309            // A previous cleanup may have removed the tree before a branch
13310            // deletion or state save failed. Unregister it if still present.
13311            git::worktree_remove_clean(repo, path).await?;
13312            return Ok(());
13313        }
13314        Err(e) => return Err(e.into()),
13315        Ok(_) => {}
13316    }
13317    if !git::is_clean(path).await? {
13318        bail!(
13319            "worktree has uncommitted work; refusing removal: {}",
13320            path.display()
13321        );
13322    }
13323    if !git::worktree_remove_clean(repo, path).await? {
13324        bail!("worktree removal refused: {}", path.display());
13325    }
13326    Ok(())
13327}
13328
13329/// Tear a run's worktrees and branches down.
13330///
13331/// `home` is where the updated `run.json` is saved (via
13332/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13333/// a housekeeping pass already has its own honest `home` handed to it, and
13334/// falling through to the global here would write back through whichever
13335/// directory some other process or test pinned into that `OnceLock` first,
13336/// not the one the caller actually resolved its `runs` and `state` from.
13337pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13338    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13339        bail!("refusing fold: {reason}");
13340    }
13341    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13342        bail!("candidate preservation incomplete; refusing cleanup");
13343    }
13344    // Serialize ordinary folding with the same starts/resumes as target-only
13345    // eviction. Otherwise a second janitor could remove the whole worktree
13346    // around a live output borrower even though target eviction was fenced.
13347    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13348    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13349    let _driver = crate::driver::maintenance(home, &state.id)?;
13350    let mut output_guards = Vec::new();
13351    for entry in std::fs::read_dir(state.worktree_root())
13352        .into_iter()
13353        .flatten()
13354        .flatten()
13355    {
13356        if entry.path().join(".git").is_file() {
13357            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13358        }
13359    }
13360    let repo = state.repo.clone();
13361    // CLI, Web and automatic cleanup all enter here, independently of
13362    // Runner::fold_losers. Protect deletion and shared config release too.
13363    let wait = repository_wait(state, None);
13364    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13365        Ok(lease) => lease,
13366        Err(e) => {
13367            state.save_under(home)?;
13368            return Err(e);
13369        }
13370    };
13371    // Confirmed merges dispose of their owned sources after durable evidence
13372    // capture. Preflight every source before removing anything.
13373    let _source_outputs = if state.status == RunStatus::Merged
13374        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13375    {
13376        verify_merged_fold_sources(state, drop_winner, home).await?
13377    } else {
13378        Vec::new()
13379    };
13380    let root = state.worktree_root();
13381    let winner = state.tally.as_ref().map(|t| t.winner);
13382    let mut removed = Vec::new();
13383
13384    for i in 0..state.candidates.len() {
13385        let c = state.candidates[i].clone();
13386        let is_winner = Some(c.label) == winner;
13387        if is_winner && !drop_winner {
13388            continue;
13389        }
13390        let existed = c.worktree.exists();
13391        // A vanished directory can still have a Git registration that keeps
13392        // its branch checked out. Remove that registration before the branch.
13393        remove_preserved_worktree(&repo, &c.worktree).await?;
13394        if existed {
13395            removed.push(c.worktree.to_string_lossy().into_owned());
13396        }
13397        // A branch handed to a later run (and its pull request) is not this
13398        // run's to delete.
13399        let handed_over = state.released_branches.contains(&c.branch);
13400        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13401            if !git::branch_delete(&repo, &c.branch).await? {
13402                bail!("candidate branch removal refused");
13403            }
13404            removed.push(c.branch.clone());
13405        }
13406        state.candidates[i].folded = true;
13407    }
13408
13409    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13410        let path = name.path();
13411        let keep = !drop_winner
13412            && winner.is_some_and(|w| {
13413                path.file_name()
13414                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13415            });
13416        if keep {
13417            continue;
13418        }
13419        remove_preserved_worktree(&repo, &path).await?;
13420        removed.push(path.to_string_lossy().into_owned());
13421    }
13422
13423    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13424    // judge worktrees, so once the loop above has cleared all of them out,
13425    // the parent is a bare directory nobody else was ever going to remove -
13426    // git only ever managed what was inside it. Left alone, one of these
13427    // accumulates per fully-folded run; the operator's own machine had 74.
13428    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13429    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13430    // so does anything a run never claimed that happens to share the bay.
13431    remove_if_empty(&root);
13432
13433    if state.enabled_worktree_config && drop_winner {
13434        // A release, not a raw disable: some sibling run in this repository
13435        // may still hold its own reference (see `git::acquire_worktree_config`),
13436        // and only the last release actually turns the setting back off.
13437        git::release_worktree_config(&repo).await.ok();
13438        state.enabled_worktree_config = false;
13439    }
13440    state.save_under(home)?;
13441    Ok(removed)
13442}
13443
13444async fn verify_merged_fold_sources(
13445    state: &RunState,
13446    drop_winner: bool,
13447    home: &Path,
13448) -> Result<Vec<std::fs::File>> {
13449    let winner = state.tally.as_ref().map(|t| t.winner);
13450    let mut heads = BTreeSet::new();
13451    for candidate in &state.candidates {
13452        if Some(candidate.label) == winner && !drop_winner {
13453            continue;
13454        }
13455        anyhow::ensure!(
13456            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13457            "candidate recovery source is outside the inventoried run root: {}",
13458            candidate.worktree.display()
13459        );
13460        match std::fs::symlink_metadata(&candidate.worktree) {
13461            Ok(_) => {}
13462            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13463            Err(error) => return Err(error.into()),
13464        }
13465        if !state.released_branches.contains(&candidate.branch)
13466            && git::branch_exists(&state.repo, &candidate.branch).await?
13467        {
13468            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13469        }
13470    }
13471    let mut cargo_guards = Vec::new();
13472    let entries = match std::fs::read_dir(state.worktree_root()) {
13473        Ok(entries) => Some(entries),
13474        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13475        Err(error) => return Err(error.into()),
13476    };
13477    for entry in entries.into_iter().flatten() {
13478        let path = entry?.path();
13479        let metadata = std::fs::symlink_metadata(&path)?;
13480        anyhow::ensure!(
13481            metadata.is_dir() && !metadata.file_type().is_symlink(),
13482            "non-directory or symlink recovery source retained: {}",
13483            path.display()
13484        );
13485        if !drop_winner
13486            && winner.is_some_and(|w| {
13487                path.file_name()
13488                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13489            })
13490        {
13491            continue;
13492        }
13493        anyhow::ensure!(
13494            path.join(".git").is_file(),
13495            "unregistered recovery source retained: {}",
13496            path.display()
13497        );
13498        anyhow::ensure!(
13499            git::git(
13500                &path,
13501                &[
13502                    "status",
13503                    "--porcelain",
13504                    "--untracked-files=all",
13505                    "--ignored",
13506                    "--",
13507                    ".",
13508                    ":(exclude)target"
13509                ]
13510            )
13511            .await?
13512            .is_empty(),
13513            "source changes or ignored recovery files retained: {}",
13514            path.display()
13515        );
13516        match std::fs::symlink_metadata(path.join("target")) {
13517            Ok(_) => {
13518                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13519            }
13520            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13521            Err(error) => return Err(error.into()),
13522        }
13523        for checkpoint in [
13524            "rebase-merge",
13525            "rebase-apply",
13526            "sequencer",
13527            "CHERRY_PICK_HEAD",
13528            "MERGE_HEAD",
13529            "REBASE_HEAD",
13530            "REVERT_HEAD",
13531            "BISECT_LOG",
13532            "AUTO_MERGE",
13533            "MERGE_MSG",
13534        ] {
13535            let location = git::git(
13536                &path,
13537                &[
13538                    "rev-parse",
13539                    "--path-format=absolute",
13540                    "--git-path",
13541                    checkpoint,
13542                ],
13543            )
13544            .await?;
13545            anyhow::ensure!(
13546                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13547                if e.kind() == std::io::ErrorKind::NotFound),
13548                "Git recovery checkpoint retained: {checkpoint}"
13549            );
13550        }
13551        heads.insert(git::rev_parse(&path, "HEAD").await?);
13552    }
13553    crate::fold_evidence::preserve(state, home, &heads).await?;
13554
13555    Ok(cargo_guards)
13556}
13557
13558/// Remove `dir` if it exists and has nothing in it.
13559///
13560/// Best-effort and silent by design: a directory that is not empty (a run
13561/// whose winner is still parked there, a stray file some other process left)
13562/// is exactly the case this must refuse, and a directory that is already gone
13563/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13564/// already refuses a non-empty directory, so the emptiness check below is
13565/// belt, not suspenders - it is what keeps this from ever attempting the
13566/// removal in the case that matters, rather than trusting `remove_dir`'s
13567/// error path to have no side effects if it ever changed.
13568fn remove_if_empty(dir: &Path) {
13569    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13570        std::fs::remove_dir(dir).ok();
13571    }
13572}
13573
13574/// Severity of the worst open finding in the last review round, for reporting.
13575pub fn worst_open(state: &RunState) -> Option<Severity> {
13576    state
13577        .reviews
13578        .last()?
13579        .reviews
13580        .iter()
13581        .flat_map(|r| r.findings.iter())
13582        .map(|f| f.severity)
13583        .max()
13584}
13585
13586/// `[worktree] setup` hides its products per worktree, which needs
13587/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13588/// worktrees, like the hook's; every place that runs setup calls this first.
13589async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13590    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13591        git::acquire_worktree_config(repo).await?;
13592        state.enabled_worktree_config = true;
13593    }
13594    Ok(())
13595}
13596
13597#[cfg(test)]
13598mod tests {
13599    #[tokio::test]
13600    async fn launcher_base_fetch_requires_remote_success() {
13601        let tmp = tempfile::tempdir().unwrap();
13602        let repo = tmp.path().join("repo");
13603        let remote = tmp.path().join("remote");
13604        std::fs::create_dir(&repo).unwrap();
13605        std::fs::create_dir(&remote).unwrap();
13606        git::git(&remote, &["init", "--bare", "-b", "main"])
13607            .await
13608            .unwrap();
13609        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13610        git::git(
13611            &repo,
13612            &[
13613                "-c",
13614                "user.name=Fixture",
13615                "-c",
13616                "user.email=fixture@example.invalid",
13617                "commit",
13618                "--allow-empty",
13619                "-m",
13620                "fixture",
13621            ],
13622        )
13623        .await
13624        .unwrap();
13625        git::git(
13626            &repo,
13627            &["remote", "add", "origin", remote.to_str().unwrap()],
13628        )
13629        .await
13630        .unwrap();
13631        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13632        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13633        assert_eq!(
13634            resolve_base(&repo, "main", "origin", None)
13635                .await
13636                .unwrap()
13637                .commit,
13638            head
13639        );
13640        #[cfg(unix)]
13641        {
13642            let policy: crate::config::Coordination = toml::from_str(
13643                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13644            ).unwrap();
13645            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13646            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13647            let held = crate::repo_lock::SeatLease::operation_with(
13648                &repo,
13649                Some(&policy),
13650                &crate::repo_lock::WaitPolicy::default(),
13651            )
13652            .await
13653            .unwrap();
13654            assert!(held.is_some());
13655            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13656            tokio::pin!(fetch);
13657            assert!(
13658                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13659                    .await
13660                    .is_err()
13661            );
13662            assert!(
13663                !repo.join(".git/FETCH_HEAD").exists(),
13664                "no fetch before admission"
13665            );
13666            drop(held);
13667            let fetched = fetch.await.unwrap();
13668            assert_eq!(fetched.commit, head);
13669            assert_eq!(fetched.remote, "origin");
13670            assert_eq!(fetched.branch, "main");
13671        }
13672        // A surviving tracking ref is not successful fetch evidence.
13673        git::git(
13674            &repo,
13675            &[
13676                "remote",
13677                "set-url",
13678                "origin",
13679                tmp.path().join("missing").to_str().unwrap(),
13680            ],
13681        )
13682        .await
13683        .unwrap();
13684        let error = resolve_base(&repo, "main", "origin", None)
13685            .await
13686            .unwrap_err()
13687            .to_string();
13688        assert!(error.contains("cannot read origin/main"));
13689        assert!(error.contains("refusing to branch off"));
13690    }
13691
13692    #[test]
13693    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13694        let leaked = vec!["chore(deps): update a crate".to_owned()];
13695        let own = "fix(daemon): apply a chosen action";
13696        assert!(should_retitle("", own, &leaked));
13697        assert!(should_retitle(
13698            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13699            own,
13700            &leaked
13701        ));
13702        assert!(should_retitle(
13703            "chore: land candidate A of run 1",
13704            own,
13705            &leaked
13706        ));
13707        assert!(should_retitle(
13708            "magi: candidate A (uncommitted work)",
13709            own,
13710            &leaked
13711        ));
13712        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13713        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13714        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13715    }
13716
13717    #[test]
13718    fn pr_merge_plan_creates_adopts_or_stops() {
13719        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13720        assert_eq!(
13721            pr_merge_plan(Ok(land::OpenPr::One {
13722                url: "u".into(),
13723                title: "t".into()
13724            })),
13725            PrPlan::Adopt {
13726                url: "u".into(),
13727                title: "t".into()
13728            }
13729        );
13730        let PrPlan::Stop(many) =
13731            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13732        else {
13733            panic!("many must stop");
13734        };
13735        assert!(many.contains('a') && many.contains('b'));
13736        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13737            panic!("a failed lookup must stop");
13738        };
13739        assert!(err.contains("bad token"));
13740    }
13741
13742    use super::*;
13743    use crate::run::GateStatus;
13744    use std::collections::BTreeMap;
13745    use std::time::Duration;
13746
13747    fn conductor() -> AgentSpec {
13748        AgentSpec {
13749            id: "conductor".to_owned(),
13750            kind: crate::config::AgentKind::Command,
13751            model: None,
13752            command: vec!["true".to_owned()],
13753            extra_args: Vec::new(),
13754            env: BTreeMap::new(),
13755            prompt_delivery: None,
13756            max_turn_secs: None,
13757        }
13758    }
13759
13760    fn spec(id: &str) -> AgentSpec {
13761        AgentSpec {
13762            id: id.to_owned(),
13763            kind: crate::config::AgentKind::Command,
13764            model: None,
13765            command: vec!["true".to_owned()],
13766            extra_args: Vec::new(),
13767            env: BTreeMap::new(),
13768            prompt_delivery: None,
13769            max_turn_secs: None,
13770        }
13771    }
13772
13773    fn ids(xs: &[&str]) -> BTreeSet<String> {
13774        xs.iter().map(|s| (*s).to_owned()).collect()
13775    }
13776
13777    #[test]
13778    fn pick_successor_skips_an_agent_another_seat_holds() {
13779        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13780        let roster = [spec("a"), spec("b"), spec("c")];
13781        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13782        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13783    }
13784
13785    #[test]
13786    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13787        // The other seat started on c but was handed to d; c is free again.
13788        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13789        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13790        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13791        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13792        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13793    }
13794
13795    #[test]
13796    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13797        let roster = [spec("a"), spec("b")];
13798        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13799        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13800        let carried = ids(&["b"]);
13801        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13802        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13803    }
13804
13805    #[test]
13806    fn pick_successor_returns_none_without_any_untried_successor() {
13807        let roster = [spec("a"), spec("b")];
13808        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13809        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13810        let carried = ids(&["a"]);
13811        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13812    }
13813
13814    #[test]
13815    fn pick_successor_rescue_avoids_another_seats_occupant() {
13816        // b is a carried failure and free; a is held by the other seat.
13817        let roster = [spec("a"), spec("b"), spec("c")];
13818        let carried = ids(&["b", "c"]);
13819        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13820        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13821    }
13822
13823    #[test]
13824    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13825        let roster = [spec("a"), spec("b"), spec("c")];
13826        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13827        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13828    }
13829
13830    #[test]
13831    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13832        let roster = [spec("a"), spec("b"), spec("c")];
13833        let failed = ids(&["a", "b", "c"]);
13834        // Rescue looks at the whole roster, once per id, then runs out.
13835        let mut tried = ids(&["b"]);
13836        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13837        assert_eq!(first.id, "a");
13838        tried.insert(first.id.clone());
13839        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13840        assert_eq!(second.id, "c");
13841        tried.insert(second.id.clone());
13842        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13843    }
13844
13845    #[test]
13846    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13847        let roster = [spec("a"), spec("b")];
13848        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13849        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13850    }
13851
13852    #[test]
13853    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13854        let roster = [spec("a"), spec("b"), spec("c")];
13855        let h = SeatHistory {
13856            failed: ids(&["a"]),
13857            last_ok: Some("b".to_owned()),
13858            last_fail: None,
13859        };
13860        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13861        // A last answerer that left the roster, or later failed, is ignored.
13862        let gone = SeatHistory {
13863            last_ok: Some("zzz".to_owned()),
13864            ..h.clone()
13865        };
13866        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13867        let failed = SeatHistory {
13868            failed: ids(&["a", "b"]),
13869            last_ok: Some("b".to_owned()),
13870            last_fail: None,
13871        };
13872        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13873    }
13874
13875    #[test]
13876    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13877        let first = SeatState::new("review-1", "alpha", 7);
13878        let next = handover_seat("review-1", "gamma", 7);
13879        assert_ne!(first.claude_session, next.claude_session);
13880    }
13881
13882    #[test]
13883    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13884        let mut state = state_with_summary("x", "y");
13885        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13886        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13887        assert_ne!(a.claude_session, b.claude_session);
13888    }
13889
13890    #[test]
13891    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13892        let roster = [spec("a"), spec("b")];
13893        let h = SeatHistory {
13894            failed: ids(&["a", "b"]),
13895            ..SeatHistory::default()
13896        };
13897        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13898        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13899        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13900    }
13901
13902    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13903    // fallback loop depends on to terminate: it must walk forward from the
13904    // seat's own position, never restart at the front of the roster, and it
13905    // must never hand back an id already tried, however many times that id
13906    // happens to appear.
13907
13908    #[test]
13909    fn failure_signature_ignores_numbers_and_paths() {
13910        assert_eq!(
13911            failure_signature("exited with Some(2) and no usable output"),
13912            failure_signature("exited with Some(137) and no usable output")
13913        );
13914        assert_eq!(
13915            failure_signature("process exited with no usable output (exit code 2)"),
13916            failure_signature("process exited with no usable output (exit code 137)")
13917        );
13918        for (old, new) in [
13919            (
13920                "exited with Some(2) and no usable output",
13921                "process exited with no usable output (exit code 2)",
13922            ),
13923            (
13924                "exited with None and no usable output",
13925                "process exited with no usable output (exit code unavailable)",
13926            ),
13927        ] {
13928            let previous = FailClass::Other(failure_signature(old));
13929            let current = FailClass::Other(failure_signature(new));
13930            assert_eq!(previous, current, "resumption retains the failure class");
13931            assert!(!should_hand_over(Some(&previous), &current));
13932        }
13933        assert_eq!(
13934            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13935            failure_signature("cannot open /var/x.txt: denied")
13936        );
13937        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13938    }
13939
13940    #[test]
13941    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13942        let other = FailClass::Other("x".into());
13943        assert!(should_hand_over(None, &FailClass::Quota));
13944        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13945        assert!(should_hand_over(
13946            Some(&FailClass::Timeout),
13947            &FailClass::Timeout
13948        ));
13949        assert!(should_hand_over(None, &other));
13950        assert!(!should_hand_over(Some(&other), &other));
13951        assert!(should_hand_over(
13952            Some(&other),
13953            &FailClass::Other("y".into())
13954        ));
13955        // A quota or timeout in between ends the run of identical failures.
13956        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13957        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13958    }
13959
13960    #[test]
13961    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13962        let a = SeatState::new("judge-1", "alpha", 7);
13963        let b = handover_seat("judge-1", "beta", 7);
13964        assert_ne!(a.claude_session, b.claude_session);
13965        assert_eq!(b.turns, 0);
13966    }
13967
13968    #[test]
13969    fn a_timeout_is_classified_apart_from_other_failures() {
13970        assert_eq!(
13971            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13972            Some(FailClass::Timeout)
13973        );
13974        assert!(matches!(
13975            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13976            Some(FailClass::Other(_))
13977        ));
13978    }
13979
13980    #[test]
13981    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13982        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13983        let tried = BTreeSet::from(["beta".to_owned()]);
13984        // beta sits at index 1; the next candidate is gamma, never alpha —
13985        // which is very likely a different candidate slot's own agent.
13986        let next = next_untried_in_roster(&roster, 1, &tried);
13987        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13988    }
13989
13990    #[test]
13991    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13992        let roster = vec![spec("alpha"), spec("beta")];
13993        let tried = BTreeSet::from(["beta".to_owned()]);
13994        // beta is the roster's last entry: nothing follows it, and alpha —
13995        // earlier in the roster, almost certainly a different candidate
13996        // slot's own agent — must not be reached by wrapping back to it.
13997        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13998    }
13999
14000    #[test]
14001    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14002        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14003        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14004        // beta (index 1) and gamma (index 2, the only entry after it) have
14005        // both been tried; alpha (index 0) never has, but it comes before
14006        // beta's own position, so there is nothing further for this seat.
14007        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14008    }
14009
14010    #[test]
14011    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14012        let roster = vec![spec("a"), spec("a"), spec("b")];
14013        let tried = BTreeSet::from(["a".to_owned()]);
14014        let next = next_untried_in_roster(&roster, 0, &tried);
14015        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14016    }
14017
14018    #[test]
14019    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14020        let roster = vec![spec("a"), spec("b")];
14021        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14022        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14023    }
14024
14025    #[test]
14026    fn remove_if_empty_only_ever_takes_a_bare_directory() {
14027        let dir = tempfile::tempdir().unwrap();
14028        let bay = dir.path().join("ffff");
14029
14030        // Not there yet: nothing to do, nothing to panic on.
14031        remove_if_empty(&bay);
14032        assert!(!bay.exists());
14033
14034        // Something still inside - the winner's worktree, or a stray file -
14035        // keeps the directory standing.
14036        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14037        remove_if_empty(&bay);
14038        assert!(bay.exists(), "non-empty directory must survive");
14039
14040        // Once the last entry is gone, so is the directory itself.
14041        std::fs::remove_dir(bay.join("cand-A")).unwrap();
14042        remove_if_empty(&bay);
14043        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14044    }
14045
14046    // `round_is_clean` is the exact decision this task fixed: a round with a
14047    // seat that never answered must not read the same as a round every seat
14048    // actually reviewed. These are deterministic and process-free by design —
14049    // the equivalent end-to-end check (a real reviewer timing out under a
14050    // live graph run) is a genuine race against wall-clock contention, and a
14051    // spawn slow enough to blow even a generous budget under a loaded test
14052    // run must not turn this specific regression check flaky.
14053
14054    #[test]
14055    fn a_full_panel_that_found_nothing_is_clean() {
14056        assert!(round_is_clean(
14057            0,
14058            true,
14059            2,
14060            2,
14061            0,
14062            IncompleteReviewPolicy::Block
14063        ));
14064    }
14065
14066    #[test]
14067    fn a_missing_seat_is_never_clean_under_the_default_policy() {
14068        assert!(!round_is_clean(
14069            0,
14070            true,
14071            1,
14072            2,
14073            0,
14074            IncompleteReviewPolicy::Block
14075        ));
14076    }
14077
14078    #[test]
14079    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14080        assert!(!round_is_clean(
14081            1,
14082            true,
14083            1,
14084            2,
14085            0,
14086            IncompleteReviewPolicy::Warn
14087        ));
14088    }
14089
14090    #[test]
14091    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14092        assert!(round_is_clean(
14093            0,
14094            true,
14095            1,
14096            2,
14097            0,
14098            IncompleteReviewPolicy::Warn
14099        ));
14100    }
14101
14102    #[test]
14103    fn a_full_panel_with_an_open_finding_is_not_clean() {
14104        assert!(!round_is_clean(
14105            1,
14106            true,
14107            2,
14108            2,
14109            0,
14110            IncompleteReviewPolicy::Block
14111        ));
14112    }
14113
14114    #[test]
14115    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14116        assert!(!round_is_clean(
14117            0,
14118            false,
14119            2,
14120            2,
14121            0,
14122            IncompleteReviewPolicy::Block
14123        ));
14124    }
14125
14126    // The stall this task closes: under the default `block` policy, a seat
14127    // missing only because it was rate limited must not force a wait for a
14128    // session limit that will not lift by the next round. `round_is_clean`
14129    // is where that quorum carve-out lives; the review loop around it never
14130    // changes what a reviewer's vote or a finding's severity means.
14131
14132    #[test]
14133    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14134        // 1 of 2 answered, and the one missing was quota'd — the exact
14135        // "review-2 rate limited (quota)" shape from the field report.
14136        assert!(round_is_clean(
14137            0,
14138            true,
14139            1,
14140            2,
14141            1,
14142            IncompleteReviewPolicy::Block
14143        ));
14144    }
14145
14146    #[test]
14147    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14148        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14149        // not a quota loss (`quota_missing` stays 0) — worth another try.
14150        assert!(!round_is_clean(
14151            0,
14152            true,
14153            1,
14154            2,
14155            0,
14156            IncompleteReviewPolicy::Block
14157        ));
14158    }
14159
14160    #[test]
14161    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14162        assert!(!round_is_clean(
14163            1,
14164            true,
14165            1,
14166            2,
14167            1,
14168            IncompleteReviewPolicy::Block
14169        ));
14170        assert!(!round_is_clean(
14171            0,
14172            false,
14173            1,
14174            2,
14175            1,
14176            IncompleteReviewPolicy::Block
14177        ));
14178    }
14179
14180    #[test]
14181    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14182        // Every seat quota'd, nobody answered: there is no panel to decide
14183        // on, so this must fall through to the existing block-and-retry
14184        // fallback rather than call an unreviewed patch clean.
14185        assert!(!round_is_clean(
14186            0,
14187            true,
14188            0,
14189            2,
14190            2,
14191            IncompleteReviewPolicy::Block
14192        ));
14193    }
14194
14195    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14196        CommandOutcome {
14197            command: "test".to_owned(),
14198            code,
14199            output_tail: String::new(),
14200            duration_ms: 0,
14201            resource_blocked,
14202        }
14203    }
14204
14205    #[cfg(unix)]
14206    #[tokio::test]
14207    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14208        ask_test_home();
14209        let temp = tempfile::tempdir().unwrap();
14210        let mut runner = runner_at(RunStatus::Gating);
14211        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14212        let commands = vec![
14213            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14214            "touch must-not-run".to_owned(),
14215        ];
14216        let (outcomes, retried, pids) = run_e2e_with_retry(
14217            &mut runner.state,
14218            &shell,
14219            &commands,
14220            temp.path(),
14221            Duration::from_secs(5),
14222            "fixture",
14223        )
14224        .await;
14225        assert_eq!(outcomes.len(), 1);
14226        assert!(outcomes[0].resource_blocked);
14227        assert!(!outcomes[0].build_failed());
14228        assert!(!retried);
14229        assert!(pids.is_empty());
14230        assert!(!temp.path().join("must-not-run").exists());
14231        assert_eq!(
14232            runner.state.resource_block.as_ref().unwrap().kind,
14233            "incremental cache admission"
14234        );
14235    }
14236
14237    #[test]
14238    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14239        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14240        assert!(
14241            !verify_inconclusive(&[outcome(Some(1), false)]),
14242            "an ordinary failure is still evidence about the patch"
14243        );
14244        assert!(verify_inconclusive(&[outcome(None, true)]));
14245        assert!(
14246            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14247            "one inconclusive outcome taints the whole batch"
14248        );
14249        assert!(!verify_inconclusive(&[]));
14250    }
14251
14252    #[tokio::test]
14253    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14254        // Alive for the first two checks, then dead - confirms the loop
14255        // actually re-polls rather than deciding once and sleeping out the
14256        // ceiling regardless.
14257        let calls = std::sync::atomic::AtomicUsize::new(0);
14258        let started = Instant::now();
14259        wait_for_pids_with(
14260            &[123],
14261            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14262            Duration::from_millis(5),
14263            Duration::from_secs(5),
14264        )
14265        .await;
14266        assert!(
14267            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14268            "must keep checking rather than deciding on the first answer"
14269        );
14270        assert!(
14271            started.elapsed() < Duration::from_secs(1),
14272            "must return the moment it is confirmed dead, not wait out the ceiling"
14273        );
14274    }
14275
14276    #[tokio::test]
14277    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14278        let started = Instant::now();
14279        wait_for_pids_with(
14280            &[123],
14281            |_| true, // never reports dead
14282            Duration::from_millis(5),
14283            Duration::from_millis(30),
14284        )
14285        .await;
14286        let elapsed = started.elapsed();
14287        assert!(
14288            elapsed >= Duration::from_millis(30),
14289            "must not give up before its own ceiling: {elapsed:?}"
14290        );
14291        assert!(
14292            elapsed < Duration::from_secs(1),
14293            "must not wait past its own ceiling either: {elapsed:?}"
14294        );
14295    }
14296
14297    #[tokio::test]
14298    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14299        let started = Instant::now();
14300        wait_for_pids_with(
14301            &[],
14302            |_| true,
14303            Duration::from_secs(5),
14304            Duration::from_secs(5),
14305        )
14306        .await;
14307        assert!(
14308            started.elapsed() < Duration::from_millis(200),
14309            "an empty pid list has nothing to confirm"
14310        );
14311    }
14312
14313    // `review_conclusion` is the exact decision the review hand-off task
14314    // fixed: a round budget spent (or a tree that stopped moving) must not
14315    // collapse into `Blocked` regardless of what verification actually
14316    // said. Deterministic and process-free for the same reason the
14317    // `round_is_clean` family above is.
14318    fn review_round(
14319        clean: bool,
14320        blocking: usize,
14321        answered: usize,
14322        expected: usize,
14323        progressed: bool,
14324        e2e_ok: bool,
14325    ) -> ReviewRound {
14326        ReviewRound {
14327            round: 1,
14328            head: "h".to_owned(),
14329            verified_head: None,
14330            verified_at: None,
14331            reviews: Vec::new(),
14332            e2e: vec![CommandOutcome {
14333                command: "test".to_owned(),
14334                code: Some(if e2e_ok { 0 } else { 1 }),
14335                output_tail: String::new(),
14336                duration_ms: 0,
14337                resource_blocked: false,
14338            }],
14339            verify_retried: false,
14340            e2e_deferred: false,
14341            e2e_defer_reason: None,
14342            fix: None,
14343            blocking,
14344            answered,
14345            expected,
14346            clean,
14347            progressed,
14348            vote_split: false,
14349            reconsideration: Vec::new(),
14350            verdict: None,
14351        }
14352    }
14353
14354    #[test]
14355    fn review_conclusion_is_none_when_nothing_has_run() {
14356        assert_eq!(review_conclusion(&[], 3), None);
14357    }
14358
14359    #[test]
14360    fn review_conclusion_is_none_while_rounds_remain() {
14361        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14362        assert_eq!(review_conclusion(&rounds, 3), None);
14363    }
14364
14365    #[test]
14366    fn review_conclusion_is_gating_once_a_round_is_clean() {
14367        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14368        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14369    }
14370
14371    #[test]
14372    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14373        let rounds = vec![
14374            review_round(false, 1, 2, 2, true, true),
14375            review_round(false, 1, 2, 2, true, true),
14376        ];
14377        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14378    }
14379
14380    #[test]
14381    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14382        let rounds = vec![
14383            review_round(false, 1, 2, 2, true, true),
14384            review_round(false, 1, 2, 2, true, false),
14385        ];
14386        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14387    }
14388
14389    #[test]
14390    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14391        // Magi never got a command to run against this round's own head — a
14392        // resource-blocked attempt, not a red one — so this must never
14393        // settle on `Blocked` the way a genuine e2e failure would. `None`
14394        // here is what tells `Runner::review_loop` to retry the check
14395        // itself rather than trust this cheap recomputation with a verdict
14396        // it cannot actually produce.
14397        let mut blocked = review_round(false, 1, 2, 2, true, false);
14398        blocked.e2e[0].resource_blocked = true;
14399        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14400        assert_eq!(review_conclusion(&rounds, 2), None);
14401    }
14402
14403    #[test]
14404    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14405        // Missing input, not a verified tree — never a hand-off candidate.
14406        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14407        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14408    }
14409
14410    #[test]
14411    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14412        let rounds = vec![
14413            review_round(false, 1, 2, 2, false, true),
14414            review_round(false, 1, 2, 2, false, true),
14415        ];
14416        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14417    }
14418
14419    fn secs(n: u64) -> Duration {
14420        Duration::from_secs(n)
14421    }
14422
14423    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14424    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14425    #[tokio::test]
14426    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14427        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14428        if std::env::var_os(CHILD).is_none() {
14429            // Other tests reconcile every run in the process-global home,
14430            // briefly taking owner locks. This ancestry test expects an idle
14431            // predecessor, so give it its own process and pinned test home.
14432            let output = std::process::Command::new(std::env::current_exe().unwrap())
14433                .args([
14434                    "--exact",
14435                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14436                    "--nocapture",
14437                ])
14438                .env(CHILD, "1")
14439                .env_remove("MAGI_WORKER_HOME")
14440                .env_remove("MAGI_WORKER_ID")
14441                .output()
14442                .unwrap();
14443            assert!(
14444                output.status.success()
14445                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14446                "{}\n{}",
14447                String::from_utf8_lossy(&output.stdout),
14448                String::from_utf8_lossy(&output.stderr)
14449            );
14450            return;
14451        }
14452        ask_test_home();
14453        let tmp = tempfile::tempdir().unwrap();
14454        let repo = tmp.path().join("repo");
14455        std::fs::create_dir_all(&repo).unwrap();
14456        init_repo(&repo);
14457        let g = |args: &[&str]| {
14458            let out = std::process::Command::new("git")
14459                .args(args)
14460                .current_dir(&repo)
14461                .quiet()
14462                .output()
14463                .unwrap();
14464            assert!(
14465                out.status.success(),
14466                "{}",
14467                String::from_utf8_lossy(&out.stderr)
14468            );
14469        };
14470        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14471        g(&["checkout", "-b", "historical"]);
14472        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14473        g(&["add", "."]);
14474        g(&["commit", "-m", "old alternative"]);
14475        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14476        g(&["checkout", "-b", "magi/task-selected", &base]);
14477        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14478        g(&["add", "."]);
14479        g(&["commit", "-m", "measurement infrastructure"]);
14480        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14481        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14482        g(&["add", "."]);
14483        g(&["commit", "-m", "dependent implementation"]);
14484        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14485        g(&["checkout", "main"]);
14486        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14487        g(&["add", "."]);
14488        g(&["commit", "-m", "unrelated base change"]);
14489        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14490        let origin = tmp.path().join("origin.git");
14491        g(&[
14492            "clone",
14493            "--bare",
14494            repo.to_str().unwrap(),
14495            origin.to_str().unwrap(),
14496        ]);
14497        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14498        g(&["fetch", "origin"]);
14499        let selection = crate::pr_recovery::Selection {
14500            url: "https://example.test/project/pull/1".into(),
14501            branch: "magi/task-selected".into(),
14502            head: head.clone(),
14503            source_head: None,
14504            source_run: None,
14505            source_proof: None,
14506            disposition: Some(crate::pr_recovery::Disposition::Review),
14507            instruction: "preserve measured behavior".into(),
14508            approved_at: jiff::Timestamp::now(),
14509        };
14510        let mut template = runner_at(RunStatus::Reviewing);
14511        template.state.repo = repo.clone();
14512        template.state.base_commit = current.clone();
14513        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14514        template.state.origin = Some(crate::run::Origin::queue("selected"));
14515        template.state.pr_head = Some(selection.branch.clone());
14516        template.state.pr_recovery = Some(selection.clone());
14517        let mut runner =
14518            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14519                .await
14520                .unwrap();
14521        let wt = runner.state.winner().unwrap().worktree.clone();
14522        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14523        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14524        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14525        assert!(runner.state.seeds.is_empty());
14526        assert_eq!(runner.state.instruction, "preserve measured behavior");
14527        runner.sync_to_base().await.unwrap();
14528        assert_eq!(
14529            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14530            "prerequisite"
14531        );
14532        assert_eq!(
14533            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14534            "measured implementation"
14535        );
14536        assert_eq!(
14537            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14538            "current main change"
14539        );
14540        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14541        assert_eq!(
14542            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14543            head
14544        );
14545
14546        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14547        let mut partial = runner_at(RunStatus::Reviewing);
14548        partial.state.repo = repo.clone();
14549        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14550        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14551            run: Some("exit 1".into()),
14552            ..Default::default()
14553        }];
14554        let selection = runner.state.pr_recovery.clone().unwrap();
14555        partial.state.pr_recovery = Some(selection.clone());
14556        partial.state.pr_head = Some(selection.branch.clone());
14557        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14558        let id = partial.state.id.clone();
14559        let error = match Runner::open_selected(
14560            &repo,
14561            partial.state,
14562            partial.roles,
14563            1,
14564            runner.state.base_commit.clone(),
14565            selection,
14566        )
14567        .await
14568        {
14569            Ok(_) => panic!("setup must fail"),
14570            Err(e) => e,
14571        };
14572        assert_eq!(
14573            error
14574                .downcast_ref::<crate::run::StartupFailure>()
14575                .unwrap()
14576                .run,
14577            id
14578        );
14579        let failed = RunState::load(&id).unwrap();
14580        assert_eq!(failed.status, RunStatus::Failed);
14581        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14582        assert!(Runner::resume(&id).is_err());
14583        runner.state.status = RunStatus::Ready;
14584        runner.state.driver_exited = true;
14585        runner.state.save().unwrap();
14586        let mut retry = runner_at(RunStatus::Reviewing);
14587        retry.state.repo = repo.clone();
14588        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14589        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14590        reselected.approved_at = jiff::Timestamp::now();
14591        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14592        retry.state.pr_head = Some(reselected.branch.clone());
14593        retry.state.pr_recovery = Some(reselected.clone());
14594        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14595            .await
14596            .unwrap();
14597        let restarted = Runner::open_selected(
14598            &repo,
14599            retry.state,
14600            retry.roles,
14601            1,
14602            runner.state.base_commit.clone(),
14603            reselected,
14604        )
14605        .await
14606        .unwrap();
14607        assert_eq!(
14608            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14609                .await
14610                .unwrap(),
14611            head
14612        );
14613        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14614        assert_eq!(
14615            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14616            head
14617        );
14618    }
14619
14620    #[tokio::test]
14621    #[cfg(unix)]
14622    async fn fallback_and_interrupted_restoration_require_coordination() {
14623        use std::os::fd::AsRawFd as _;
14624        ask_test_home();
14625        let tmp = tempfile::tempdir().unwrap();
14626        let mut runner = registered_runner(tmp.path()).await;
14627        runner.state.config.graph.timeout_resource_wait = 1;
14628        let winner = runner.state.candidates[0].clone();
14629        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14630        let leftover = winner.worktree.join("leftover");
14631        std::fs::write(&leftover, "untrusted edits").unwrap();
14632        let next = tmp.path().join("next-fixer");
14633        let mut failed = spec("first");
14634        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14635        let mut successor = spec("second");
14636        successor.command = vec![
14637            "sh".into(),
14638            "-c".into(),
14639            format!("touch '{}'; echo done", next.display()),
14640        ];
14641        let attempts = vec![
14642            (failed.clone(), "first".into()),
14643            (successor.clone(), "second".into()),
14644        ];
14645        let lock_path = runner.state.repo.join(".git/updater.lock");
14646        let external = std::fs::File::create(&lock_path).unwrap();
14647        assert_eq!(
14648            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14649            0
14650        );
14651        let artifacts = tmp.path().join("artifacts");
14652        let build = |spec, seat| SeatJob {
14653            spec,
14654            seat,
14655            cwd: winner.worktree.clone(),
14656            prompt: "fixture".into(),
14657            timeout: Duration::from_secs(10),
14658            allow_write: true,
14659            sessions: false,
14660            artifacts: artifacts.clone(),
14661            stem: "fix".into(),
14662            handover: None,
14663        };
14664        let (_, _, out) = runner
14665            .ask_fixer_among(
14666                attempts.clone(),
14667                "graft",
14668                None,
14669                Some((&winner.worktree, &head)),
14670                build,
14671            )
14672            .await;
14673        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14674        assert!(leftover.exists(), "contention must prevent clean");
14675        assert!(!next.exists());
14676        assert!(runner.state.handovers.is_empty());
14677        // Early interrupted-repair rollback uses the same protected route.
14678        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14679            from_run: "earlier".into(),
14680            generation: 1,
14681            mode: crate::recovery::Mode::VerifyRepair,
14682            reasons: Vec::new(),
14683            reviewed_head: None,
14684            rounds: Vec::new(),
14685            open_findings: Vec::new(),
14686            evidence: None,
14687            criteria_inherited: true,
14688            repair: crate::recovery::Repair::Started {
14689                before: head.clone(),
14690            },
14691        });
14692        assert!(
14693            runner
14694                .recovery_repair(&winner)
14695                .await
14696                .unwrap_err()
14697                .is::<crate::repo_lock::ResourceBlocked>()
14698        );
14699        assert!(matches!(
14700            runner.state.recovery.as_ref().unwrap().repair,
14701            crate::recovery::Repair::Started { .. }
14702        ));
14703        assert!(leftover.exists());
14704        drop(external);
14705        runner.recovery_repair(&winner).await.unwrap();
14706        assert!(!leftover.exists());
14707        assert!(matches!(
14708            runner.state.recovery.as_ref().unwrap().repair,
14709            crate::recovery::Repair::Done { .. }
14710        ));
14711        // A hard lock-open failure also leaves HEAD and untracked work intact.
14712        std::fs::remove_file(&lock_path).unwrap();
14713        std::fs::create_dir(&lock_path).unwrap();
14714        std::fs::write(&leftover, "untrusted edits").unwrap();
14715        let (_, _, out) = runner
14716            .ask_fixer_among(
14717                attempts,
14718                "graft",
14719                None,
14720                Some((&winner.worktree, &head)),
14721                build,
14722            )
14723            .await;
14724        assert!(matches!(out, AgentOutcome::Failed(_)));
14725        assert!(leftover.exists());
14726        assert!(!next.exists());
14727        assert_eq!(
14728            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14729            head
14730        );
14731    }
14732
14733    #[cfg(unix)]
14734    async fn registered_runner(root: &Path) -> Runner {
14735        repository_runner(root, crate::config::test_coordination()).await
14736    }
14737
14738    async fn repository_runner(
14739        root: &Path,
14740        coordination: Option<crate::config::Coordination>,
14741    ) -> Runner {
14742        ask_test_home();
14743        let repo = root.join("repo");
14744        std::fs::create_dir_all(&repo).unwrap();
14745        init_repo(&repo);
14746        if coordination.is_some() {
14747            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14748        }
14749        let mut runner = runner_at(RunStatus::Prep);
14750        runner.state.repo = repo.clone();
14751        runner.state.config.coordination = coordination;
14752        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14753        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14754        runner.state.config.graph.advise = false;
14755        runner.roles.implementers = vec![spec("implement")];
14756        runner.roles.judges = vec![spec("judge")];
14757        runner.prep().await.unwrap();
14758        runner
14759    }
14760
14761    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14762
14763    #[tokio::test]
14764    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14765        let tmp = tempfile::tempdir().unwrap();
14766        let mut runner = repository_runner(tmp.path(), None).await;
14767        let prompts = runner.state.config.prompts.clone();
14768        let run = runner.state.id.clone();
14769        let ctx = WaveCtx {
14770            carry_seats: false,
14771            pauses: None,
14772            run: &run,
14773            node: "advise",
14774            prompts: &prompts,
14775            cache: None,
14776            round: None,
14777        };
14778        let jobs: Vec<_> = ["alpha", "beta"]
14779            .into_iter()
14780            .map(|id| SeatJob {
14781                spec: scripted(id, AUTH_OUTAGE),
14782                seat: SeatState::new(id, id, 7),
14783                prompt: "propose".into(),
14784                cwd: runner.state.candidates[0].worktree.clone(),
14785                timeout: Duration::from_secs(10),
14786                allow_write: false,
14787                sessions: false,
14788                artifacts: agent::artifacts_dir(&runner.state.dir()),
14789                stem: id.into(),
14790                handover: None,
14791            })
14792            .collect();
14793        let mut losses = Vec::new();
14794        let results = ask_wave_with(
14795            jobs.clone(),
14796            runner.sem.clone(),
14797            2,
14798            &[],
14799            &ctx,
14800            &mut losses,
14801            &mut runner.state,
14802            &|_: &str| Ok(()),
14803        )
14804        .await;
14805        assert!(
14806            results
14807                .iter()
14808                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14809        );
14810        assert_eq!(runner.state.quota.len(), 2);
14811        assert!(
14812            runner
14813                .state
14814                .quota
14815                .iter()
14816                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14817        );
14818        assert!(!runner.state.advise_attempted);
14819        runner.stop_for_resource().unwrap();
14820        assert_eq!(runner.state.status, RunStatus::Blocked);
14821        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14822        assert_eq!(
14823            runner.state.resource_block.as_ref().unwrap().kind,
14824            "agent authentication"
14825        );
14826        // A machine loss followed by an ordinary failure is not evidence
14827        // that the whole node was unavailable, and must not earn a refund.
14828        runner.state.resource_block = None;
14829        runner.state.quota.clear();
14830        let mut mixed = jobs;
14831        mixed[1].spec.command = vec!["false".into()];
14832        let results = ask_wave_with(
14833            mixed,
14834            runner.sem.clone(),
14835            0,
14836            &[],
14837            &ctx,
14838            &mut losses,
14839            &mut runner.state,
14840            &|_: &str| Ok(()),
14841        )
14842        .await;
14843        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14844        assert!(runner.state.resource_block.is_none());
14845        assert_eq!(losses.len(), 1);
14846    }
14847
14848    #[tokio::test]
14849    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14850        let tmp = tempfile::tempdir().unwrap();
14851        let mut runner = repository_runner(tmp.path(), None).await;
14852        let implementer = scripted("implement", AUTH_OUTAGE);
14853        runner.roles.implementers = vec![implementer.clone()];
14854        runner.state.config.agents = vec![implementer];
14855        runner.implement().await.unwrap();
14856        assert_eq!(runner.state.status, RunStatus::Blocked);
14857        assert_eq!(runner.state.quota.len(), 1);
14858        assert_eq!(
14859            runner.state.quota[0].cause,
14860            agent::QuotaCause::Authentication
14861        );
14862        let candidate = &runner.state.candidates[0];
14863        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14864        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14865    }
14866
14867    #[tokio::test]
14868    #[cfg(unix)]
14869    async fn parked_implementer_persists_completed_work() {
14870        for registered in [false, true] {
14871            let tmp = tempfile::tempdir().unwrap();
14872            let mut runner = registered_runner(tmp.path()).await;
14873            if !registered {
14874                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14875            }
14876            let marker = tmp.path().join("started");
14877            let release = tmp.path().join("release");
14878            let mut implementer = spec("implement");
14879            implementer.command = vec![
14880                "sh".into(),
14881                "-c".into(),
14882                format!(
14883                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14884                    marker.display(),
14885                    release.display()
14886                ),
14887            ];
14888            runner.roles.implementers = vec![implementer.clone()];
14889            runner.state.config.agents = vec![implementer];
14890            let pause = runner.pause.clone();
14891            let (result, ()) = tokio::join!(runner.implement(), async {
14892                tokio::time::timeout(Duration::from_secs(10), async {
14893                    while !marker.exists() {
14894                        tokio::time::sleep(Duration::from_millis(10)).await;
14895                    }
14896                })
14897                .await
14898                .unwrap();
14899                pause.park();
14900                std::fs::write(&release, "go").unwrap();
14901            });
14902            result.unwrap();
14903            let candidate = &runner.state.candidates[0];
14904            assert!(candidate.commits > 0, "{candidate:?}");
14905            assert!(candidate.summary.contains("saved-result"));
14906            assert_eq!(candidate.files, 1);
14907            let before = candidate.commits;
14908            runner.implement().await.unwrap();
14909            assert_eq!(runner.state.candidates[0].commits, before);
14910        }
14911    }
14912
14913    #[tokio::test]
14914    #[cfg(unix)]
14915    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14916        let tmp = tempfile::tempdir().unwrap();
14917        let mut runner = registered_runner(tmp.path()).await;
14918        let winner = runner.state.candidates[0].clone();
14919        let cache = tmp.path().join("cache");
14920        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14921        runner.state.config.graph.timeout_resource_wait = 0;
14922        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14923        let crate::cache::AcquireOutcome::Acquired(held) =
14924            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14925        else {
14926            panic!("cache must be free")
14927        };
14928        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14929            from_run: "earlier".into(),
14930            generation: 1,
14931            mode: crate::recovery::Mode::VerifyRepair,
14932            reasons: Vec::new(),
14933            reviewed_head: None,
14934            rounds: Vec::new(),
14935            open_findings: Vec::new(),
14936            evidence: None,
14937            criteria_inherited: true,
14938            repair: crate::recovery::Repair::NotStarted,
14939        });
14940        runner.recovery_repair(&winner).await.unwrap();
14941        assert!(matches!(
14942            runner.state.recovery.as_ref().unwrap().repair,
14943            crate::recovery::Repair::WaitingForRestore { .. }
14944        ));
14945        drop(held);
14946        runner.state.resource_block = None;
14947        runner.state.config.graph.timeout_resource_wait = 1;
14948        runner.recovery_repair(&winner).await.unwrap();
14949        assert!(matches!(
14950            runner.state.recovery.as_ref().unwrap().repair,
14951            crate::recovery::Repair::Done { .. }
14952        ));
14953        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14954    }
14955
14956    #[tokio::test]
14957    #[cfg(unix)]
14958    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14959        use std::os::fd::AsRawFd as _;
14960        for graft in [false, true] {
14961            let tmp = tempfile::tempdir().unwrap();
14962            let mut runner = registered_runner(tmp.path()).await;
14963            runner.state.config.graph.timeout_resource_wait = 1;
14964            let winner = runner.state.candidates[0].clone();
14965            let marker = tmp.path().join("fixer-started");
14966            let mut fixer = spec("fixer");
14967            fixer.command = vec![
14968                "sh".into(),
14969                "-c".into(),
14970                format!("echo started >> '{}'; echo done", marker.display()),
14971            ];
14972            runner.roles.fixer = Some(vec![fixer]);
14973            if graft {
14974                let mut loser = winner.clone();
14975                loser.label = 'B';
14976                loser.index = 1;
14977                runner.state.candidates.push(loser);
14978                runner.state.config.graph.graft_runner_up = true;
14979                runner.state.tally = Some(Tally {
14980                    first_choice: BTreeMap::new(),
14981                    borda: BTreeMap::new(),
14982                    winner: winner.label,
14983                    rankings: 1,
14984                    unanimous_initial: true,
14985                    deliberated: false,
14986                    changed_votes: 0,
14987                    unanimous_final: true,
14988                    tie_break: None,
14989                    judges: 1,
14990                    present: 1,
14991                    quorum: 1,
14992                    met_quorum: true,
14993                    uncontested: None,
14994                });
14995                runner.state.judgements.push(crate::run::Judgement {
14996                    judge: 1,
14997                    seat: "judge-1".into(),
14998                    agent: "judge".into(),
14999                    ranking: vec![winner.label, 'B'],
15000                    reasons: BTreeMap::new(),
15001                    confidence: None,
15002                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15003                    order: vec![0, 1],
15004                    failed: None,
15005                    duration_ms: 0,
15006                });
15007            } else {
15008                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15009                    from_run: "earlier".into(),
15010                    generation: 1,
15011                    mode: crate::recovery::Mode::VerifyRepair,
15012                    reasons: Vec::new(),
15013                    reviewed_head: None,
15014                    rounds: Vec::new(),
15015                    open_findings: Vec::new(),
15016                    evidence: None,
15017                    criteria_inherited: true,
15018                    repair: crate::recovery::Repair::NotStarted,
15019                });
15020            }
15021            let reader = crate::repo_lock::SeatLease::for_checkout(
15022                &winner.worktree,
15023                false,
15024                crate::config::test_coordination().as_ref(),
15025            )
15026            .await
15027            .unwrap();
15028            let pause = runner.pause.clone();
15029            let (result, ()) = tokio::join!(
15030                async {
15031                    if graft {
15032                        runner.graft().await
15033                    } else {
15034                        runner.recovery_repair(&winner).await
15035                    }
15036                },
15037                async {
15038                    tokio::time::sleep(Duration::from_millis(100)).await;
15039                    pause.park();
15040                }
15041            );
15042            result.unwrap();
15043            assert!(!marker.exists(), "a blocked seat must not spawn");
15044            // Exercise the persisted state, as a binary replacement/resume does.
15045            runner.state = RunState::load(&runner.state.id).unwrap();
15046            if graft {
15047                assert_eq!(
15048                    runner.state.graft.as_ref().unwrap().outcome,
15049                    GraftOutcome::WaitingForRestore
15050                );
15051            } else {
15052                assert!(matches!(
15053                    runner.state.recovery.as_ref().unwrap().repair,
15054                    crate::recovery::Repair::WaitingForRestore { .. }
15055                ));
15056            }
15057            drop(reader);
15058            runner.pause = Pause::new();
15059            runner.state.resource_block = None;
15060            // Deferred restoration must still respect an external lock owner.
15061            let external =
15062                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15063            assert_eq!(
15064                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15065                0
15066            );
15067            let leftover = winner.worktree.join("leftover");
15068            std::fs::write(&leftover, "earlier fallback edits").unwrap();
15069            let result = if graft {
15070                runner.graft().await
15071            } else {
15072                runner.recovery_repair(&winner).await
15073            };
15074            assert!(
15075                result
15076                    .unwrap_err()
15077                    .is::<crate::repo_lock::ResourceBlocked>()
15078            );
15079            assert!(leftover.exists());
15080            assert!(!marker.exists());
15081            drop(external);
15082            runner.state.resource_block = None;
15083            let result = if graft {
15084                runner.graft().await
15085            } else {
15086                runner.recovery_repair(&winner).await
15087            };
15088            result.unwrap();
15089            assert!(!leftover.exists(), "restore must precede retry");
15090            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15091            if graft {
15092                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15093            } else {
15094                assert!(matches!(
15095                    runner.state.recovery.as_ref().unwrap().repair,
15096                    crate::recovery::Repair::Done { .. }
15097                ));
15098            }
15099        }
15100    }
15101
15102    #[tokio::test]
15103    #[cfg(unix)]
15104    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15105        use std::os::fd::AsRawFd as _;
15106        let tmp = tempfile::tempdir().unwrap();
15107        let mut runner = registered_runner(tmp.path()).await;
15108        runner.state.config.graph.timeout_resource_wait = 1;
15109        let winner = runner.state.candidates[0].clone();
15110        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15111        runner.state.tally = Some(Tally {
15112            first_choice: BTreeMap::new(),
15113            borda: BTreeMap::new(),
15114            winner: winner.label,
15115            rankings: 0,
15116            unanimous_initial: false,
15117            deliberated: false,
15118            changed_votes: 0,
15119            unanimous_final: false,
15120            tie_break: None,
15121            judges: 0,
15122            present: 0,
15123            quorum: 0,
15124            met_quorum: true,
15125            uncontested: Some("fixture".into()),
15126        });
15127        runner.state.graft = Some(GraftRecord {
15128            pre: pre.clone(),
15129            post: None,
15130            lines: None,
15131            limit: None,
15132            agent: None,
15133            outcome: GraftOutcome::Running,
15134            requested: Vec::new(),
15135            adoption: None,
15136            adoption_error: None,
15137        });
15138        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15139        git::commit_all(&winner.worktree, "unfinished graft")
15140            .await
15141            .unwrap();
15142        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15143        let leftover = winner.worktree.join("leftover");
15144        std::fs::write(&leftover, "untrusted edits").unwrap();
15145        let path = runner.state.repo.join(".git/updater.lock");
15146        let external = std::fs::File::create(&path).unwrap();
15147        assert_eq!(
15148            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15149            0
15150        );
15151        assert!(
15152            runner
15153                .graft()
15154                .await
15155                .unwrap_err()
15156                .is::<crate::repo_lock::ResourceBlocked>()
15157        );
15158        assert!(matches!(
15159            runner.state.graft.as_ref().unwrap().outcome,
15160            GraftOutcome::Running
15161        ));
15162        assert_eq!(
15163            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15164            unfinished
15165        );
15166        assert!(leftover.exists());
15167        drop(external);
15168        std::fs::remove_file(&path).unwrap();
15169        std::fs::create_dir(&path).unwrap();
15170        assert!(runner.graft().await.is_err());
15171        assert!(matches!(
15172            runner.state.graft.as_ref().unwrap().outcome,
15173            GraftOutcome::Running
15174        ));
15175        assert_eq!(
15176            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15177            unfinished
15178        );
15179        assert!(leftover.exists());
15180        std::fs::remove_dir(&path).unwrap();
15181        runner.graft().await.unwrap();
15182        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15183        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15184        assert!(!leftover.exists());
15185    }
15186
15187    #[tokio::test]
15188    #[cfg(unix)]
15189    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15190        use std::os::fd::AsRawFd as _;
15191        let tmp = tempfile::tempdir().unwrap();
15192        let mut runner = registered_runner(tmp.path()).await;
15193        runner.state.prep_complete = Some(false);
15194        runner.state.config.graph.timeout_resource_wait = 1;
15195        runner.state.save().unwrap();
15196        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15197        assert_eq!(
15198            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15199            0
15200        );
15201        runner.execute().await.unwrap();
15202        let disk = RunState::load(&runner.state.id).unwrap();
15203        assert!(disk.driver_exited);
15204        assert_eq!(disk.driver_pid, None);
15205        assert_eq!(runner.state.status, RunStatus::Blocked);
15206        assert_eq!(
15207            runner.state.resource_block.as_ref().unwrap().kind,
15208            "repository coordination"
15209        );
15210        assert!(runner.state.prep_failure.is_none());
15211        assert!(runner.state.seats.is_empty());
15212        runner.pause.park();
15213        // Include headroom for Git subprocesses under parallel test load;
15214        // the repository contention deadline itself remains one second.
15215        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15216            .await
15217            .unwrap()
15218            .unwrap();
15219        assert!(runner.state.parked);
15220        assert!(runner.state.prep_failure.is_none());
15221        let disk = RunState::load(&runner.state.id).unwrap();
15222        assert!(disk.driver_exited);
15223        assert_eq!(disk.driver_pid, None);
15224    }
15225
15226    #[tokio::test]
15227    #[cfg(unix)]
15228    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15229        use std::os::fd::AsRawFd as _;
15230        let tmp = tempfile::tempdir().unwrap();
15231        let mut runner = registered_runner(tmp.path()).await;
15232        runner.state.prep_complete = Some(false);
15233        runner.state.config.graph.timeout_resource_wait = 30;
15234        runner.state.save().unwrap();
15235        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15236        assert_eq!(
15237            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15238            0
15239        );
15240        assert!(
15241            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15242                .await
15243                .is_err()
15244        );
15245        let disk = RunState::load(&runner.state.id).unwrap();
15246        assert!(disk.driver_exited);
15247        assert_eq!(disk.driver_pid, None);
15248        assert!(disk.status.resumable());
15249        assert!(runner.state.driver_exited);
15250        assert_eq!(runner.state.driver_pid, None);
15251        runner.state.save().unwrap();
15252        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15253    }
15254
15255    #[tokio::test]
15256    #[cfg(unix)]
15257    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15258        let tmp = tempfile::tempdir().unwrap();
15259        let mut runner = registered_runner(tmp.path()).await;
15260        let winner = runner.state.candidates[0].clone();
15261        let cache = tmp.path().join("cache");
15262        let started = tmp.path().join("pre-gate-started");
15263        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15264        runner.state.config.verify.gate =
15265            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15266        let home = crate::run::home();
15267        let owner =
15268            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15269        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15270            crate::cache::AcquireOutcome::Acquired(g) => g,
15271            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15272        };
15273        let mut gate = Box::pin(runner.run_gate(&winner));
15274        assert!(
15275            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15276                .await
15277                .is_err()
15278        );
15279        assert!(!started.exists(), "pre-gate must wait for the cache too");
15280        // Simulate the wave that already owns the cache entering invoke.
15281        // It must obtain the editor reservation and finish without waiting
15282        // for the gate that is queued behind its cache.
15283        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15284        let editor = crate::repo_lock::SeatLease::checkout_with(
15285            &winner.worktree,
15286            true,
15287            crate::config::test_coordination().as_ref(),
15288            &wait,
15289        )
15290        .await
15291        .unwrap();
15292        drop(editor);
15293        guard.release();
15294        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15295            .await
15296            .unwrap()
15297            .unwrap();
15298        drop(gate);
15299        assert!(outcomes.iter().all(CommandOutcome::ok));
15300        assert!(started.exists());
15301    }
15302
15303    #[tokio::test]
15304    #[cfg(unix)]
15305    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15306        use std::os::fd::AsRawFd as _;
15307
15308        ask_test_home();
15309        let tmp = tempfile::tempdir().unwrap();
15310        let repo = tmp.path().join("repo");
15311        std::fs::create_dir_all(&repo).unwrap();
15312        init_repo(&repo);
15313        let common = repo.join(".git");
15314        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15315        let mut runner = runner_at(RunStatus::Prep);
15316        runner.state.repo = repo.clone();
15317        runner.state.config.coordination = crate::config::test_coordination();
15318        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15319        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15320        runner.state.config.graph.advise = false;
15321        runner.roles.implementers = vec![spec("implement")];
15322        runner.roles.judges = vec![spec("judge")];
15323        runner.prep().await.unwrap();
15324        let winner = runner.state.candidates[0].clone();
15325        // Emulate edits already left by a completed seat. The fixture CLI
15326        // only reports completion; it performs no repository operations.
15327        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15328        let completed = tmp.path().join("seat-completed");
15329        runner.roles.implementers[0].command = vec![
15330            "sh".into(),
15331            "-c".into(),
15332            format!("touch '{}'; echo completed", completed.display()),
15333        ];
15334        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15335        assert_eq!(
15336            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15337            0
15338        );
15339        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15340        let mut implement = Box::pin(runner.implement());
15341        tokio::select! {
15342            result = &mut implement => panic!("rescue must wait: {result:?}"),
15343            result = tokio::time::timeout(Duration::from_secs(10), async {
15344                while !completed.exists() {
15345                    tokio::time::sleep(Duration::from_millis(20)).await;
15346                }
15347            }) => result.unwrap(),
15348        }
15349        assert!(
15350            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15351                .await
15352                .is_err()
15353        );
15354        assert_eq!(
15355            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15356            before
15357        );
15358        assert_eq!(
15359            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15360            0
15361        );
15362        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15363            .await
15364            .unwrap()
15365            .unwrap();
15366        drop(implement);
15367        assert_ne!(
15368            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15369            before
15370        );
15371        assert!(git::is_clean(&winner.worktree).await.unwrap());
15372
15373        let started = tmp.path().join("pre-gate-started");
15374        let proceed = tmp.path().join("pre-gate-proceed");
15375        let checked = tmp.path().join("gate-ran");
15376        runner.state.config.verify.pre_gate = vec![format!(
15377            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15378            started.display(),
15379            proceed.display()
15380        )];
15381        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15382        assert_eq!(
15383            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15384            0
15385        );
15386        let mut gate = Box::pin(runner.run_gate(&winner));
15387        assert!(
15388            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15389                .await
15390                .is_err()
15391        );
15392        assert!(!started.exists());
15393        assert!(!checked.exists());
15394        assert!(git::is_clean(&winner.worktree).await.unwrap());
15395        assert_eq!(
15396            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15397            0
15398        );
15399        tokio::select! {
15400            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15401            result = tokio::time::timeout(Duration::from_secs(10), async {
15402                while !started.exists() {
15403                    tokio::time::sleep(Duration::from_millis(20)).await;
15404                }
15405            }) => result.unwrap(),
15406        }
15407        assert_eq!(
15408            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15409            -1,
15410            "mechanical edits must hold the real lock"
15411        );
15412        std::fs::write(proceed, "ready").unwrap();
15413        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15414            .await
15415            .unwrap()
15416            .unwrap();
15417        drop(gate);
15418        assert!(outcomes.iter().all(CommandOutcome::ok));
15419        assert!(checked.exists());
15420        assert!(runner.state.pre_gate_commit.is_some());
15421        assert!(git::is_clean(&winner.worktree).await.unwrap());
15422        assert_eq!(
15423            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15424            0,
15425            "gate completion must release coordination"
15426        );
15427        drop(outsider);
15428        std::fs::remove_file(common.join("updater.lock")).unwrap();
15429        std::fs::create_dir(common.join("updater.lock")).unwrap();
15430        std::fs::remove_file(&started).unwrap();
15431        std::fs::remove_file(&checked).unwrap();
15432        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15433        let error = runner.run_gate(&winner).await.unwrap_err();
15434        assert!(
15435            error
15436                .to_string()
15437                .contains("acquire mandatory repository lock")
15438        );
15439        assert!(
15440            !started.exists(),
15441            "failed acquisition must prevent pre-gate"
15442        );
15443        assert!(!checked.exists(), "failed acquisition must prevent gate");
15444        assert!(git::is_clean(&winner.worktree).await.unwrap());
15445        assert_eq!(
15446            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15447            head
15448        );
15449    }
15450
15451    #[tokio::test]
15452    #[cfg(unix)]
15453    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15454        use std::os::fd::AsRawFd as _;
15455        use std::time::Duration;
15456
15457        ask_test_home();
15458        let tmp = tempfile::tempdir().unwrap();
15459        let repo = tmp.path().join("repo");
15460        std::fs::create_dir_all(&repo).unwrap();
15461        init_repo(&repo);
15462        let common = repo.join(".git");
15463        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15464        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15465        assert_eq!(
15466            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15467            0
15468        );
15469        let mut runner = runner_at(RunStatus::Prep);
15470        runner.state.repo = repo.clone();
15471        runner.state.config.coordination = crate::config::test_coordination();
15472        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15473        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15474        runner.state.config.graph.advise = true;
15475        runner.state.config.graph.advisors = 1;
15476        runner.roles.implementers = vec![spec("implement")];
15477        runner.roles.judges = vec![spec("judge")];
15478        let root = runner.state.worktree_root();
15479        let started = tmp.path().join("setup-started");
15480        let proceed = tmp.path().join("setup-proceed");
15481        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15482            run: Some(format!(
15483                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15484                started.display(),
15485                proceed.display()
15486            )),
15487            ..Default::default()
15488        }];
15489        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15490        let config = git::git(&repo, &["config", "--local", "--list"])
15491            .await
15492            .unwrap();
15493        let mut prep = Box::pin(runner.prep());
15494        assert!(
15495            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15496                .await
15497                .is_err()
15498        );
15499        assert!(
15500            !root.exists(),
15501            "contention must prevent even checkout creation"
15502        );
15503        assert!(!started.exists(), "contention must prevent setup");
15504        assert_eq!(
15505            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15506            branches
15507        );
15508        assert_eq!(
15509            git::git(&repo, &["config", "--local", "--list"])
15510                .await
15511                .unwrap(),
15512            config
15513        );
15514        drop(lock);
15515        tokio::select! {
15516            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15517            result = tokio::time::timeout(Duration::from_secs(10), async {
15518                while !started.exists() {
15519                    tokio::time::sleep(Duration::from_millis(20)).await;
15520                }
15521            }) => result.unwrap(),
15522        }
15523        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15524        assert_eq!(
15525            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15526            -1
15527        );
15528        assert!(
15529            tokio::time::timeout(
15530                Duration::from_millis(100),
15531                crate::repo_lock::SeatLease::for_checkout(
15532                    &repo,
15533                    true,
15534                    crate::config::test_coordination().as_ref()
15535                )
15536            )
15537            .await
15538            .is_err(),
15539            "preparation must also exclude local editors"
15540        );
15541        std::fs::write(proceed, "ready").unwrap();
15542        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15543            .await
15544            .unwrap()
15545            .unwrap();
15546        drop(prep);
15547        assert_eq!(runner.state.prep_complete, Some(true));
15548        assert!(root.join("judge-1").exists());
15549        assert!(root.join("advisor-1").exists());
15550        assert_eq!(
15551            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15552            0
15553        );
15554    }
15555
15556    #[tokio::test]
15557    #[cfg(unix)]
15558    async fn preparation_lock_failure_prevents_repository_changes() {
15559        ask_test_home();
15560        let tmp = tempfile::tempdir().unwrap();
15561        let repo = tmp.path().join("repo");
15562        std::fs::create_dir_all(&repo).unwrap();
15563        init_repo(&repo);
15564        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15565        // A directory at the required lock path makes opening the real lock
15566        // fail, rather than merely exercising an invalid registration.
15567        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15568        let mut runner = runner_at(RunStatus::Prep);
15569        runner.state.repo = repo.clone();
15570        runner.state.config.coordination = crate::config::test_coordination();
15571        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15572        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15573        runner.roles.implementers = vec![spec("implement")];
15574        let error = runner.prep().await.unwrap_err();
15575        assert!(
15576            error
15577                .to_string()
15578                .contains("acquire mandatory repository lock")
15579        );
15580        assert!(!runner.state.worktree_root().exists());
15581        assert!(runner.state.candidates.is_empty());
15582        assert!(!repo.join(".git/config.worktree").exists());
15583        assert_eq!(
15584            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15585            "* main"
15586        );
15587    }
15588
15589    #[tokio::test]
15590    async fn preparation_records_creation_intent_before_git_can_fail() {
15591        ask_test_home();
15592        let tmp = tempfile::tempdir().unwrap();
15593        let repo = tmp.path().join("repo");
15594        std::fs::create_dir_all(&repo).unwrap();
15595        init_repo(&repo);
15596        let mut runner = runner_at(RunStatus::Prep);
15597        runner.state.repo = repo.clone();
15598        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15599        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15600        runner.state.config.blind.commit_msg_hook = false;
15601        runner.roles.implementers = vec![spec("one")];
15602        let label = blind::assign_labels(1, runner.state.seed)[0];
15603        let branch = runner.state.branch_for(label);
15604        // A conflicting branch makes Git refuse creation before a candidate
15605        // can be registered. The durable intent must still identify it.
15606        git::git(&repo, &["branch", &branch]).await.unwrap();
15607        assert!(runner.prep().await.is_err());
15608        let saved = RunState::load(&runner.state.id).unwrap();
15609        assert!(saved.candidates.is_empty());
15610        let event = saved
15611            .events
15612            .iter()
15613            .find(|e| e.node == "prep-intent")
15614            .unwrap();
15615        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15616        assert_eq!(intent["branch"], branch);
15617        assert_eq!(intent["start"], runner.state.base_commit);
15618        assert_eq!(
15619            intent["worktree"],
15620            runner
15621                .state
15622                .worktree_root()
15623                .join(format!("cand-{label}"))
15624                .to_string_lossy()
15625                .as_ref()
15626        );
15627        assert_eq!(intent["picks"], serde_json::json!([]));
15628    }
15629
15630    #[tokio::test]
15631    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15632        ask_test_home();
15633        let tmp = tempfile::tempdir().unwrap();
15634        let repo = tmp.path().join("repo");
15635        std::fs::create_dir_all(&repo).unwrap();
15636        init_repo(&repo);
15637        let mut runner = runner_at(RunStatus::Prep);
15638        runner.state.repo = repo.clone();
15639        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15640        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15641        runner.state.sources.pick_commits.push(refs::Input {
15642            token: "missing".into(),
15643            sha: "0".repeat(40),
15644            provenance: refs::Provenance::ExplicitCli,
15645        });
15646        assert!(runner.prep().await.is_err());
15647        assert!(runner.state.candidates.is_empty());
15648        assert!(!runner.state.worktree_root().exists());
15649    }
15650
15651    #[tokio::test]
15652    async fn prose_and_legacy_inference_never_import_into_candidates() {
15653        ask_test_home();
15654        let tmp = tempfile::tempdir().unwrap();
15655        let repo = tmp.path().join("repo");
15656        std::fs::create_dir_all(&repo).unwrap();
15657        init_repo(&repo);
15658        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15659        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15660            .await
15661            .unwrap();
15662        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15663        git::git(&repo, &["add", "."]).await.unwrap();
15664        git::git(&repo, &["commit", "-m", "historical implementation"])
15665            .await
15666            .unwrap();
15667        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15668        git::git(&repo, &["checkout", "main"]).await.unwrap();
15669        for instruction in [
15670            format!("Example: magi/history/A at {source}"),
15671            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15672            format!("Never import magi/history/A or {source}"),
15673            format!("Conversation answer: I mentioned {source} only as background"),
15674        ] {
15675            let mut runner = runner_at(RunStatus::Prep);
15676            runner.state.repo = repo.clone();
15677            runner.state.base_commit = base.clone();
15678            runner.state.instruction = instruction.clone();
15679            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15680            runner.state.config.blind.commit_msg_hook = false;
15681            runner.state.seeds = vec![refs::Seed {
15682                token: "magi/history/A".into(),
15683                sha: source.clone(),
15684                kind: refs::SeedKind::Unmerged,
15685                branch: true,
15686                detail: String::new(),
15687            }];
15688            runner.roles.implementers = vec![spec("one")];
15689            assert_eq!(seeded_instruction(&runner.state), instruction);
15690            // Model an old record with inferred seeds but no structured input field.
15691            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15692            legacy.as_object_mut().unwrap().remove("sources");
15693            runner.state = serde_json::from_value(legacy).unwrap();
15694            assert_eq!(runner.state.sources, refs::Sources::default());
15695            runner.state.save().unwrap();
15696            runner.state = RunState::load(&runner.state.id).unwrap();
15697            runner.prep().await.unwrap();
15698            for candidate in &runner.state.candidates {
15699                assert_eq!(
15700                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15701                    base
15702                );
15703                assert!(!candidate.worktree.join("unwanted").exists());
15704            }
15705        }
15706    }
15707
15708    #[tokio::test]
15709    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15710        ask_test_home();
15711        let tmp = tempfile::tempdir().unwrap();
15712        let repo = tmp.path().join("repo");
15713        std::fs::create_dir_all(&repo).unwrap();
15714        init_repo(&repo);
15715        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15716        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15717        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15718        git::git(&repo, &["add", "."]).await.unwrap();
15719        git::git(&repo, &["commit", "-m", "required seed"])
15720            .await
15721            .unwrap();
15722        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15723        git::git(&repo, &["checkout", "main"]).await.unwrap();
15724        let mut runner = runner_at(RunStatus::Prep);
15725        runner.state.repo = repo.clone();
15726        runner.state.base_commit = base.clone();
15727        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15728        runner.state.config.graph.advise = false;
15729        runner.state.config.blind.commit_msg_hook = false;
15730        runner.state.sources =
15731            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15732                .await
15733                .unwrap();
15734        runner.roles.implementers = vec![spec("one"), spec("two")];
15735        runner.roles.judges = vec![spec("judge")];
15736        let label = blind::assign_labels(2, runner.state.seed)[0];
15737        let branch = runner.state.branch_for(label);
15738        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15739        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15740            .await
15741            .unwrap();
15742        runner.state.candidates.push(Candidate {
15743            index: 0,
15744            label,
15745            agent: "one".into(),
15746            branch,
15747            worktree,
15748            summary: String::new(),
15749            stat: String::new(),
15750            files: 0,
15751            commits: 0,
15752            empty: false,
15753            failed: None,
15754            verified_noop: None,
15755            duration_ms: 0,
15756            folded: false,
15757        });
15758        runner.state.save().unwrap();
15759        // Reload the checkpoint immediately before its first cherry-pick.
15760        runner.state = RunState::load(&runner.state.id).unwrap();
15761        assert_eq!(runner.state.prep_complete, Some(false));
15762        runner.prep().await.unwrap();
15763        assert_eq!(runner.state.prep_complete, Some(true));
15764        assert_eq!(runner.state.candidates.len(), 2);
15765        for candidate in &runner.state.candidates {
15766            assert_eq!(
15767                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15768                git::tree_of(&repo, &seed).await.unwrap()
15769            );
15770            assert_eq!(
15771                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15772                "required commit"
15773            );
15774        }
15775        assert!(runner.state.worktree_root().join("judge-1").exists());
15776        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15777        runner.prep().await.unwrap();
15778        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15779
15780        runner.state.prep_complete = Some(false);
15781        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15782            run: Some(format!(
15783                "test -f '{}'",
15784                tmp.path().join("dependency-ready").display()
15785            )),
15786            ..Default::default()
15787        }];
15788        assert!(runner.prep().await.is_err());
15789        assert!(runner.state.prep_failure.is_some());
15790        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15791        runner.state = RunState::load(&runner.state.id).unwrap();
15792        assert!(
15793            runner.prep().await.is_err(),
15794            "correcting the cause alone must not auto-retry"
15795        );
15796        let failed_at = runner.state.events.last().unwrap().at;
15797        assert!(
15798            runner.retry_preparation(failed_at).is_err(),
15799            "old approvals cannot unlock the hold"
15800        );
15801        let mut task = crate::queue::Task::new(
15802            "retry".into(),
15803            "retry setup".into(),
15804            repo.clone(),
15805            crate::queue::Source::Human,
15806        );
15807        task.start(runner.state.id.clone());
15808        task.hold_machine(Some("setup failed".into()));
15809        task.release_operator("cli", None);
15810        runner
15811            .retry_preparation(task.resumption.as_ref().unwrap().at)
15812            .unwrap();
15813        runner.prep().await.unwrap();
15814        assert!(runner.state.prep_failure.is_none());
15815        assert_eq!(runner.state.prep_complete, Some(true));
15816        assert_eq!(runner.state.candidates.len(), 2);
15817        assert!(runner.state.worktree_root().join("judge-1").exists());
15818        for candidate in &runner.state.candidates {
15819            assert_eq!(
15820                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15821                git::tree_of(&repo, &seed).await.unwrap()
15822            );
15823        }
15824    }
15825
15826    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15827        let mut heads = Vec::new();
15828        for candidate in candidates {
15829            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15830        }
15831        heads
15832    }
15833
15834    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15835        ask_test_home();
15836        let tmp = tempfile::tempdir().unwrap();
15837        let repo = tmp.path().join("repo");
15838        std::fs::create_dir(&repo).unwrap();
15839        init_repo(&repo);
15840        let mut runner = runner_at(RunStatus::Reviewing);
15841        runner.state.repo = repo.clone();
15842        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15843        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15844        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15845            let branch = format!("candidate-{label}");
15846            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15847            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15848                .await
15849                .unwrap();
15850            if label == 'B' {
15851                std::fs::write(worktree.join("change"), "winner").unwrap();
15852                git::rescue_commit(&worktree, "candidate change")
15853                    .await
15854                    .unwrap();
15855            }
15856            runner.state.candidates.push(Candidate {
15857                index,
15858                label,
15859                agent: "alpha".into(),
15860                branch,
15861                worktree,
15862                summary: "completed reply".into(),
15863                stat: String::new(),
15864                files: usize::from(label == 'B'),
15865                commits: usize::from(label == 'B'),
15866                empty: label == 'A',
15867                failed: None,
15868                verified_noop: None,
15869                duration_ms: 1,
15870                folded: false,
15871            });
15872        }
15873        runner.state.judge_skipped = true;
15874        runner.state.tally = Some(Tally {
15875            first_choice: BTreeMap::new(),
15876            borda: BTreeMap::new(),
15877            winner: 'B',
15878            rankings: 0,
15879            unanimous_initial: false,
15880            deliberated: false,
15881            changed_votes: 0,
15882            unanimous_final: false,
15883            tie_break: None,
15884            judges: 0,
15885            present: 0,
15886            quorum: 0,
15887            met_quorum: true,
15888            uncontested: Some("only B produced a change".into()),
15889        });
15890        runner.state.save().unwrap();
15891        (tmp, runner)
15892    }
15893
15894    #[tokio::test]
15895    #[cfg(unix)]
15896    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15897        for (solo, failed_checkpoint) in
15898            [(true, false), (false, false), (true, true), (false, true)]
15899        {
15900            let (_tmp, mut runner) = preservation_resume_fixture().await;
15901            if solo {
15902                runner.state.candidates.retain(|c| c.label == 'A');
15903            }
15904            runner.state.candidates[0].empty = false;
15905            runner.state.candidates[0].summary.clear();
15906            runner.state.tally = None;
15907            runner.state.judge_skipped = false;
15908            runner.state.status = RunStatus::Implementing;
15909            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15910            let mut implementer = spec("alpha");
15911            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15912            runner.roles.implementers = vec![implementer];
15913            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15914            if failed_checkpoint {
15915                std::fs::create_dir_all(&patch).unwrap();
15916            }
15917            runner.state.save().unwrap();
15918            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15919            drop(pressure);
15920            runner.state = RunState::load(&runner.state.id).unwrap();
15921            assert_eq!(runner.state.status, RunStatus::Blocked);
15922            assert!(runner.state.interrupted_implementations.contains(&'A'));
15923            assert_eq!(runner.state.candidates[0].commits, 0);
15924            assert!(!runner.state.candidates[0].empty);
15925            assert!(runner.state.candidates[0].failed.is_none());
15926            if failed_checkpoint {
15927                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15928                std::fs::remove_dir(patch).unwrap();
15929            }
15930            runner.state.resource_block = None;
15931            runner.state.config.disk.min_free_bytes = 0;
15932            runner.roles.implementers[0].command = vec![
15933                "sh".into(),
15934                "-c".into(),
15935                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15936            ];
15937            runner.implement().await.unwrap();
15938            assert!(runner.state.interrupted_implementations.is_empty());
15939            assert!(runner.state.candidate_preservation.is_empty());
15940            assert_eq!(runner.state.candidates[0].commits, 1);
15941            assert!(!runner.state.candidates[0].empty);
15942            assert!(
15943                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15944                    .unwrap()
15945                    .contains("retried empty interrupted seat")
15946            );
15947            if !solo {
15948                assert_eq!(runner.state.candidates[1].commits, 1);
15949            }
15950        }
15951    }
15952
15953    #[tokio::test]
15954    #[cfg(unix)]
15955    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15956        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15957            let (_tmp, mut runner) = preservation_resume_fixture().await;
15958            runner.state.candidates.retain(|c| c.label == 'A');
15959            runner.state.candidates[0].empty = false;
15960            runner.state.candidates[0].summary.clear();
15961            runner.state.tally = None;
15962            runner.state.judge_skipped = false;
15963            runner.state.status = RunStatus::Implementing;
15964            // Force only the real post-spawn watchdog, without failing the
15965            // separate admission check. Existing edits predate its measurement.
15966            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15967            let wt = runner.state.candidates[0].worktree.clone();
15968            std::fs::write(
15969                wt.join("interrupted-source"),
15970                "retain implementation work\n",
15971            )
15972            .unwrap();
15973            if committed {
15974                git::rescue_commit(&wt, "implementation checkpoint")
15975                    .await
15976                    .unwrap();
15977            }
15978            let lock = git::git(
15979                &wt,
15980                &[
15981                    "rev-parse",
15982                    "--path-format=absolute",
15983                    "--git-path",
15984                    "index.lock",
15985                ],
15986            )
15987            .await
15988            .unwrap();
15989            if block_rescue {
15990                std::fs::write(&lock, "fixture-owned lock").unwrap();
15991            }
15992            let mut implementer = spec("alpha");
15993            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15994            runner.roles.implementers = vec![implementer];
15995            runner.state.save().unwrap();
15996            let result = runner.implement().await;
15997            drop(pressure);
15998            assert_eq!(result.is_err(), block_rescue);
15999            assert_eq!(
16000                runner.state.resource_block.as_ref().unwrap().kind,
16001                "disk pressure"
16002            );
16003            assert!(
16004                runner
16005                    .state
16006                    .events
16007                    .iter()
16008                    .any(|e| e.message.contains("agent execution interrupted"))
16009            );
16010            assert!(
16011                runner
16012                    .state
16013                    .dir()
16014                    .join("artifacts/candidate-A-recovery/staged.patch")
16015                    .exists()
16016            );
16017            if block_rescue {
16018                let saved = RunState::load(&runner.state.id).unwrap();
16019                assert!(saved.candidate_preservation.contains_key(&'A'));
16020                assert!(!saved.candidates[0].empty);
16021                assert_eq!(
16022                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16023                    "retain implementation work\n"
16024                );
16025                std::fs::remove_file(lock).unwrap();
16026                runner.state = saved;
16027            } else {
16028                assert_eq!(runner.state.status, RunStatus::Blocked);
16029                assert_eq!(runner.state.candidates[0].commits, 1);
16030            }
16031            // Recovery must save the candidate before deciding whether another
16032            // implementation is needed. No implementer is available on retry.
16033            runner.roles.implementers.clear();
16034            runner.state.config.disk.min_free_bytes = 0;
16035            runner.state.resource_block = None;
16036            runner.implement().await.unwrap();
16037            assert!(runner.state.candidate_preservation.is_empty());
16038            assert_eq!(runner.state.candidates[0].commits, 1);
16039            assert!(!runner.state.candidates[0].empty);
16040            assert!(git::is_clean(&wt).await.unwrap());
16041            assert!(
16042                crate::run::read_artifact(&runner.state, "cand-A.patch")
16043                    .unwrap()
16044                    .contains("retain implementation work")
16045            );
16046        }
16047    }
16048
16049    #[tokio::test]
16050    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16051        let (_tmp, mut runner) = preservation_resume_fixture().await;
16052        // Schema 15 already confirmed the unfolded no-change result.
16053        runner.implement().await.unwrap();
16054        assert!(runner.state.candidate_preservation.is_empty());
16055        assert!(
16056            !runner
16057                .state
16058                .dir()
16059                .join("artifacts/candidate-A-recovery")
16060                .exists()
16061        );
16062        runner.fold_losers().await.unwrap();
16063        assert!(!runner.state.candidates[0].worktree.exists());
16064        for schema in [14, crate::run::SCHEMA] {
16065            runner.state.schema = schema;
16066            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16067            runner.state.save().unwrap();
16068            runner.state = RunState::load(&runner.state.id).unwrap();
16069            runner.implement().await.unwrap();
16070            assert!(runner.state.candidate_preservation.is_empty());
16071            assert!(runner.state.candidates[0].folded);
16072            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16073        }
16074    }
16075
16076    #[tokio::test]
16077    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16078        let (_tmp, mut runner) = preservation_resume_fixture().await;
16079        runner.state.candidates.retain(|c| c.label == 'A');
16080        runner.state.tally = None;
16081        runner.state.judge_skipped = false;
16082        let candidate = &mut runner.state.candidates[0];
16083        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16084        candidate.failed = Some("deployment validation unavailable".into());
16085        std::fs::write(
16086            candidate.worktree.join("repair"),
16087            "future launcher policy\n",
16088        )
16089        .unwrap();
16090        runner
16091            .state
16092            .candidate_preservation
16093            .insert('A', "awaiting persistence".into());
16094        runner.persist_candidates().await.unwrap();
16095        let candidate = &runner.state.candidates[0];
16096        assert!(!candidate.empty);
16097        assert_eq!(candidate.commits, 1);
16098        assert_eq!(candidate.files, 1);
16099        assert!(candidate.failed.is_none());
16100        assert!(candidate.summary.contains("EPERM"));
16101        assert!(
16102            crate::run::read_artifact(&runner.state, "cand-A.patch")
16103                .unwrap()
16104                .contains("future launcher policy")
16105        );
16106        assert!(runner.state.candidate_preservation.is_empty());
16107    }
16108
16109    #[tokio::test]
16110    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16111        let (_tmp, mut runner) = preservation_resume_fixture().await;
16112        let wt = runner.state.candidates[0].worktree.clone();
16113        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16114        runner.state.schema = 14;
16115        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16116        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16117        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16118        for _ in 0..2 {
16119            assert!(runner.implement().await.is_err());
16120            assert_eq!(runner.state.status, RunStatus::Blocked);
16121            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16122            assert!(
16123                runner.state.candidates[0].empty,
16124                "old verdict must remain unusable"
16125            );
16126            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16127            assert!(runner.state.judge_skipped);
16128            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16129            assert!(
16130                crate::run::read_artifact(&runner.state, "cand-A.patch")
16131                    .unwrap()
16132                    .contains("owned edits")
16133            );
16134            assert!(wt.exists());
16135            assert_eq!(
16136                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16137                    .await
16138                    .unwrap(),
16139                1
16140            );
16141            runner.state = RunState::load(&runner.state.id).unwrap();
16142        }
16143    }
16144
16145    #[tokio::test]
16146    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16147        let (_tmp, mut runner) = preservation_resume_fixture().await;
16148        let wt = runner.state.candidates[0].worktree.clone();
16149        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16150        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16151        let index_lock = PathBuf::from(
16152            git::git(
16153                &wt,
16154                &[
16155                    "rev-parse",
16156                    "--path-format=absolute",
16157                    "--git-path",
16158                    "index.lock",
16159                ],
16160            )
16161            .await
16162            .unwrap(),
16163        );
16164        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16165        runner.state.schema = 14;
16166        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16167        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16168        assert!(runner.implement().await.is_err());
16169        assert!(runner.after_implement().is_err());
16170        assert!(runner.fold_losers().await.is_err());
16171        assert_eq!(
16172            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16173            "failed seat edits"
16174        );
16175        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16176        runner.state = RunState::load(&runner.state.id).unwrap();
16177        std::fs::remove_file(index_lock).unwrap();
16178        // Recovery finds changes that the old uncontested result never judged.
16179        assert!(runner.implement().await.is_err());
16180        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16181        assert!(runner.state.candidates[0].empty);
16182        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16183        assert!(
16184            crate::run::read_artifact(&runner.state, "cand-A.patch")
16185                .unwrap()
16186                .contains("failed seat edits")
16187        );
16188        assert!(wt.exists());
16189    }
16190
16191    #[tokio::test]
16192    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16193        let (_tmp, mut runner) = preservation_resume_fixture().await;
16194        let repo = runner.state.repo.clone();
16195        let common = git::git(
16196            &repo,
16197            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16198        )
16199        .await
16200        .unwrap();
16201        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16202        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16203        assert!(runner.fold_losers().await.is_err());
16204        assert!(!runner.state.candidates[0].worktree.exists());
16205        assert!(!runner.state.candidates[0].folded);
16206        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16207        runner.state = RunState::load(&runner.state.id).unwrap();
16208        std::fs::remove_file(branch_lock).unwrap();
16209        let judge = runner.state.worktree_root().join("judge-1");
16210        git::worktree_add_detached(&repo, &judge, "main")
16211            .await
16212            .unwrap();
16213        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16214        runner.roles.judges = vec![spec("judge")];
16215        assert!(runner.fold_losers().await.is_err());
16216        runner.state = RunState::load(&runner.state.id).unwrap();
16217        assert!(
16218            runner.state.candidates[0].folded,
16219            "candidate progress saved before judge failure"
16220        );
16221        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16222        assert!(judge.join("uncommitted").exists());
16223        std::fs::remove_file(judge.join("uncommitted")).unwrap();
16224        runner.fold_losers().await.unwrap();
16225        assert!(!judge.exists());
16226        runner.fold_losers().await.unwrap();
16227    }
16228
16229    #[tokio::test]
16230    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16231        let (_tmp, mut runner) = preservation_resume_fixture().await;
16232        runner.state.candidates.retain(|c| c.label == 'B');
16233        runner.state.tally = None;
16234        runner.state.judge_skipped = false;
16235        runner.state.prep_complete = Some(true);
16236        runner.state.config.graph.advise = false;
16237        runner.state.config.graph.selected_implementer = Some("alpha".into());
16238        let wt = runner.state.candidates[0].worktree.clone();
16239        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16240        runner
16241            .state
16242            .candidate_preservation
16243            .insert('B', "awaiting persistence".into());
16244        let index_lock = git::git(
16245            &wt,
16246            &[
16247                "rev-parse",
16248                "--path-format=absolute",
16249                "--git-path",
16250                "index.lock",
16251            ],
16252        )
16253        .await
16254        .unwrap();
16255        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16256        runner.state.save().unwrap();
16257        assert!(runner.execute_graph().await.is_err());
16258        assert_eq!(runner.state.status, RunStatus::Blocked);
16259        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16260        std::fs::remove_file(index_lock).unwrap();
16261        // Park at the next graph boundary: this tests the real selected-seat
16262        // hold guard without dispatching reviewers or publishing anything.
16263        for _ in 0..2 {
16264            runner.state = RunState::load(&runner.state.id).unwrap();
16265            runner.pause.park();
16266            runner.execute_graph().await.unwrap();
16267            assert_eq!(runner.state.status, RunStatus::Implementing);
16268            assert!(runner.state.candidate_preservation.is_empty());
16269            assert_eq!(runner.state.candidates[0].commits, 2);
16270            assert!(runner.state.tally.is_none());
16271        }
16272        runner.implement().await.unwrap();
16273        assert_eq!(runner.state.status, RunStatus::Judging);
16274        // A genuine selected-implementer hold must still require release.
16275        runner.state.status = RunStatus::Blocked;
16276        runner.state.candidates[0].failed = Some("implementation failed".into());
16277        runner.state.save().unwrap();
16278        let error = runner.execute_graph().await.unwrap_err();
16279        assert!(
16280            error
16281                .to_string()
16282                .contains("selected implementation is held")
16283        );
16284    }
16285
16286    #[tokio::test]
16287    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16288        let (_tmp, mut runner) = preservation_resume_fixture().await;
16289        let repo = runner.state.repo.clone();
16290        let candidate = runner.state.candidates[0].clone();
16291        // Remove only the disposable fixture directory, leaving Git's linked
16292        // worktree registration and branch intact, as in a vanished checkout.
16293        let registered = git::worktree_holding(&repo, &candidate.branch)
16294            .await
16295            .unwrap()
16296            .unwrap();
16297        assert_eq!(
16298            registered.canonicalize().unwrap(),
16299            candidate.worktree.canonicalize().unwrap()
16300        );
16301        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16302        assert_eq!(
16303            git::worktree_holding(&repo, &candidate.branch)
16304                .await
16305                .unwrap(),
16306            Some(registered)
16307        );
16308        assert!(
16309            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16310            "the stale registration initially keeps the branch checked out"
16311        );
16312        runner.fold_losers().await.unwrap();
16313        assert!(runner.state.candidates[0].folded);
16314        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16315        assert_eq!(
16316            git::worktree_holding(&repo, &candidate.branch)
16317                .await
16318                .unwrap(),
16319            None
16320        );
16321        runner.state = RunState::load(&runner.state.id).unwrap();
16322        runner.fold_losers().await.unwrap();
16323    }
16324
16325    #[tokio::test]
16326    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16327        let (_tmp, mut runner) = preservation_resume_fixture().await;
16328        let repo = runner.state.repo.clone();
16329        let candidate = runner.state.candidates[0].clone();
16330        let winner = runner.state.candidates[1].clone();
16331        let registered = git::worktree_holding(&repo, &candidate.branch)
16332            .await
16333            .unwrap()
16334            .unwrap();
16335        assert_eq!(
16336            registered.canonicalize().unwrap(),
16337            candidate.worktree.canonicalize().unwrap()
16338        );
16339        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16340        assert_eq!(
16341            git::worktree_holding(&repo, &candidate.branch)
16342                .await
16343                .unwrap(),
16344            Some(registered)
16345        );
16346        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16347
16348        for _ in 0..2 {
16349            fold_run(&mut runner.state, false, &crate::run::home())
16350                .await
16351                .unwrap();
16352            runner.state = RunState::load(&runner.state.id).unwrap();
16353            assert!(runner.state.candidates[0].folded);
16354            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16355            assert_eq!(
16356                git::worktree_holding(&repo, &candidate.branch)
16357                    .await
16358                    .unwrap(),
16359                None
16360            );
16361            assert!(!runner.state.candidates[1].folded);
16362            assert!(winner.worktree.exists());
16363            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16364        }
16365    }
16366
16367    #[tokio::test]
16368    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16369        ask_test_home();
16370        let tmp = tempfile::tempdir().unwrap();
16371        let repo = tmp.path().join("repo");
16372        std::fs::create_dir(&repo).unwrap();
16373        init_repo(&repo);
16374        let mut runner = runner_at(RunStatus::Implementing);
16375        runner.state.repo = repo.clone();
16376        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16377        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16378        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16379            let branch = format!("candidate-{label}");
16380            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16381            git::worktree_add_branch(&repo, &wt, &branch, "main")
16382                .await
16383                .unwrap();
16384            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16385            git::git(&wt, &["add", "README.md"]).await.unwrap();
16386            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16387            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16388            runner.state.candidates.push(Candidate {
16389                index,
16390                label,
16391                agent: "alpha".to_owned(),
16392                branch,
16393                worktree: wt,
16394                summary: "completed implementation".to_owned(),
16395                stat: String::new(),
16396                files: 0,
16397                commits: 0,
16398                empty: false,
16399                failed: None,
16400                verified_noop: None,
16401                duration_ms: 42,
16402                folded: false,
16403            });
16404            runner
16405                .state
16406                .candidate_preservation
16407                .insert(label, "awaiting persistence".to_owned());
16408        }
16409        let wt = runner.state.candidates[0].worktree.clone();
16410        let index_lock = git::git(
16411            &wt,
16412            &[
16413                "rev-parse",
16414                "--path-format=absolute",
16415                "--git-path",
16416                "index.lock",
16417            ],
16418        )
16419        .await
16420        .unwrap();
16421        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16422        runner.state.save().unwrap();
16423        assert!(runner.persist_candidates().await.is_err());
16424        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16425        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16426        assert_eq!(
16427            runner.state.candidates[1].commits, 1,
16428            "other results are still collected"
16429        );
16430        assert!(
16431            !runner.state.candidates[0].empty,
16432            "failed rescue is not no change"
16433        );
16434        assert_eq!(
16435            runner.state.candidates[0].summary,
16436            "completed implementation"
16437        );
16438        assert!(runner.after_implement().is_err());
16439        let home = tmp.path().join("fold-home");
16440        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16441        assert!(wt.join("new-file").exists());
16442        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16443        assert!(recovery.join("staged.patch").exists());
16444        assert!(recovery.join("unstaged.patch").exists());
16445        assert_eq!(
16446            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16447            "untracked edits\n"
16448        );
16449        runner.state.begin_pass();
16450        runner.state.end_pass_parked(ParkCause::Upgrade {
16451            from: "1.0".into(),
16452            to: Some("1.1".into()),
16453        });
16454        runner.state.save().unwrap();
16455        runner.state = RunState::load(&runner.state.id).unwrap();
16456        assert!(matches!(
16457            runner.state.passes.last().unwrap().end,
16458            Some(crate::run::PassEnd::Parked {
16459                cause: ParkCause::Upgrade { .. }
16460            })
16461        ));
16462        std::fs::remove_file(index_lock).unwrap();
16463        // No implementer is configured: a persistence retry cannot dispatch a seat.
16464        runner.implement().await.unwrap();
16465        assert!(runner.state.candidate_preservation.is_empty());
16466        assert!(
16467            runner
16468                .state
16469                .candidates
16470                .iter()
16471                .all(|c| c.commits == 1 && !c.empty)
16472        );
16473        assert!(git::is_clean(&wt).await.unwrap());
16474        assert!(
16475            crate::run::read_artifact(&runner.state, "cand-A.patch")
16476                .unwrap()
16477                .contains("later tracked edits")
16478        );
16479    }
16480
16481    #[tokio::test]
16482    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16483        ask_test_home();
16484        let tmp = tempfile::tempdir().unwrap();
16485        let repo = tmp.path().join("repo");
16486        std::fs::create_dir(&repo).unwrap();
16487        init_repo(&repo);
16488        let wt = tmp.path().join("candidate");
16489        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16490            .await
16491            .unwrap();
16492        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16493        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16494        assert!(wt.join("uncommitted").exists());
16495        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16496        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16497        let mut runner = runner_at(RunStatus::Implementing);
16498        runner.state.repo = repo.clone();
16499        runner.state.base_commit = "missing-base".to_owned();
16500        runner.state.candidates.push(Candidate {
16501            index: 0,
16502            label: 'A',
16503            agent: "alpha".into(),
16504            branch: "candidate".into(),
16505            worktree: wt.clone(),
16506            summary: "completed".into(),
16507            stat: String::new(),
16508            files: 0,
16509            commits: 0,
16510            empty: false,
16511            failed: None,
16512            verified_noop: None,
16513            duration_ms: 1,
16514            folded: false,
16515        });
16516        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16517        assert!(
16518            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16519                .await
16520                .is_err()
16521        );
16522        assert!(!runner.state.candidates[0].folded);
16523        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16524        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16525        runner
16526            .state
16527            .candidate_preservation
16528            .insert('A', "pending".into());
16529        assert!(runner.persist_candidates().await.is_err());
16530        assert!(!runner.state.candidates[0].empty);
16531        assert!(!runner.state.candidates[0].folded);
16532        assert!(wt.exists());
16533        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16534        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16535        std::fs::create_dir_all(&patch_path).unwrap();
16536        assert!(
16537            runner.persist_candidates().await.is_err(),
16538            "artifact failure stays pending"
16539        );
16540        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16541        assert!(!runner.state.candidates[0].empty);
16542        std::fs::remove_dir(patch_path).unwrap();
16543        runner.persist_candidates().await.unwrap();
16544        assert!(
16545            runner.state.candidates[0].empty,
16546            "only successful inspection confirms no change"
16547        );
16548    }
16549
16550    fn init_repo(dir: &Path) {
16551        let run = |args: &[&str]| {
16552            let out = std::process::Command::new("git")
16553                .args(args)
16554                .current_dir(dir)
16555                .quiet()
16556                .output()
16557                .expect("spawn git");
16558            assert!(
16559                out.status.success(),
16560                "git {args:?} failed: {}",
16561                String::from_utf8_lossy(&out.stderr)
16562            );
16563        };
16564        run(&["init", "-b", "main"]);
16565        run(&["config", "user.name", "magi test"]);
16566        run(&["config", "user.email", "magi@example.com"]);
16567        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16568        run(&["add", "-A"]);
16569        run(&["commit", "-m", "init"]);
16570    }
16571
16572    // `settle_questions` is what closes the ghost the phone showed: a run's
16573    // seat asked something, the run then ended, and nothing was left to
16574    // abandon the question it left `open`. `HOME` is a process-wide
16575    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16576    // first time it runs in the binary — every test below still reaches the
16577    // same directory whichever call won, and each gets its own run id from
16578    // `RunState::new`, so they never collide there.
16579    fn ask_test_home() {
16580        crate::run::pin_test_home();
16581    }
16582
16583    #[test]
16584    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16585        ask_test_home();
16586        let mut runner = runner_at(RunStatus::Prep);
16587        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16588        // A blocked temporary output path makes the terminal save fail while
16589        // preserving the already published reservation.
16590        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16591        let error = guard
16592            .finish(
16593                &mut runner.state,
16594                Err(
16595                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16596                        .into(),
16597                ),
16598            )
16599            .unwrap_err();
16600        assert_eq!(
16601            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16602            std::io::ErrorKind::PermissionDenied
16603        );
16604        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16605        let message = format!("{error:#}");
16606        assert!(message.contains("original refusal"));
16607        assert!(!message.contains("could not save terminal"));
16608        assert_eq!(
16609            error.chain().count(),
16610            2,
16611            "only startup attribution wraps the original error"
16612        );
16613        let saved = RunState::load(&runner.state.id).unwrap();
16614        assert_eq!(saved.status, RunStatus::Blocked);
16615        assert!(!saved.startup.unwrap().complete);
16616    }
16617
16618    #[test]
16619    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16620        ask_test_home();
16621        let mut runner = runner_at(RunStatus::Prep);
16622        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16623        let id = runner.state.id.clone();
16624        drop(guard);
16625        let saved = RunState::load(&id).unwrap();
16626        assert_eq!(saved.status, RunStatus::Blocked);
16627        assert!(!saved.startup.as_ref().unwrap().complete);
16628        assert!(
16629            saved
16630                .startup
16631                .unwrap()
16632                .failure
16633                .unwrap()
16634                .contains("interrupted")
16635        );
16636        assert!(Runner::resume(&id).is_err());
16637    }
16638
16639    #[test]
16640    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16641        ask_test_home();
16642        let mut runner = runner_at(RunStatus::Prep);
16643        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16644        let error = guard
16645            .finish(
16646                &mut runner.state,
16647                Err(
16648                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16649                        .into(),
16650                ),
16651            )
16652            .unwrap_err();
16653        assert_eq!(
16654            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16655            std::io::ErrorKind::PermissionDenied
16656        );
16657        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16658        let mut task = crate::queue::Task::new(
16659            "review".into(),
16660            "criteria".into(),
16661            runner.state.repo.clone(),
16662            crate::queue::Source::Human,
16663        );
16664        task.record_startup_failure(failure.clone());
16665        task.record_startup_failure(failure.clone());
16666        assert_eq!(task.startup_failures.len(), 1);
16667        assert!(task.runs.is_empty());
16668        assert_eq!(task.attempts, 0);
16669    }
16670
16671    #[cfg(unix)]
16672    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16673        ask_test_home();
16674        let tmp = tempfile::tempdir().unwrap();
16675        let repo = tmp.path().join("repo");
16676        std::fs::create_dir(&repo).unwrap();
16677        init_repo(&repo);
16678        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16679        let origin = tmp.path().join("origin.git");
16680        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16681            .await
16682            .unwrap();
16683        git::git(
16684            &repo,
16685            &["remote", "add", "origin", origin.to_str().unwrap()],
16686        )
16687        .await
16688        .unwrap();
16689        git::git(&repo, &["checkout", "-b", "fix/retained"])
16690            .await
16691            .unwrap();
16692        std::fs::write(repo.join("retained"), "retained change").unwrap();
16693        git::git(&repo, &["add", "."]).await.unwrap();
16694        git::git(&repo, &["commit", "-m", "retained implementation"])
16695            .await
16696            .unwrap();
16697        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16698        git::git(&repo, &["checkout", "main"]).await.unwrap();
16699        std::fs::write(repo.join("base-change"), "new base").unwrap();
16700        git::git(&repo, &["add", "."]).await.unwrap();
16701        git::git(&repo, &["commit", "-m", "base moved"])
16702            .await
16703            .unwrap();
16704        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16705        git::git(&repo, &["checkout", "fix/retained"])
16706            .await
16707            .unwrap();
16708        let mut runner = runner_at(RunStatus::Voting);
16709        runner.state.repo = repo.clone();
16710        runner.state.base_commit = base;
16711        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16712        runner.state.config.disk.min_free_bytes = 0;
16713        runner.state.config.graph.review_rounds = 1;
16714        runner.state.config.verify.e2e = vec!["true".into()];
16715        runner.state.config.verify.gate = vec!["true".into()];
16716        runner.state.candidates = vec![Candidate {
16717            index: 0,
16718            label: 'A',
16719            agent: "fixture".into(),
16720            branch: "fix/retained".into(),
16721            worktree: repo.clone(),
16722            summary: String::new(),
16723            stat: String::new(),
16724            files: 1,
16725            commits: 1,
16726            empty: false,
16727            failed: None,
16728            verified_noop: None,
16729            duration_ms: 0,
16730            folded: false,
16731        }];
16732        runner.state.tally = Some(Tally {
16733            first_choice: BTreeMap::new(),
16734            borda: BTreeMap::new(),
16735            winner: 'A',
16736            rankings: 0,
16737            unanimous_initial: true,
16738            deliberated: false,
16739            changed_votes: 0,
16740            unanimous_final: true,
16741            tie_break: None,
16742            judges: 0,
16743            present: 0,
16744            quorum: 0,
16745            met_quorum: true,
16746            uncontested: Some("review-only".into()),
16747        });
16748        let mut task = crate::queue::Task::new(
16749            "fixture".into(),
16750            "fixture".into(),
16751            repo,
16752            crate::queue::Source::Human,
16753        );
16754        task.link_run(&runner.state.id);
16755        crate::queue::Queue::open().put(&mut task).unwrap();
16756        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16757        runner.state.base_sync = Some(BaseSync {
16758            tip: String::new(),
16759            behind: 1,
16760            attempts: BASE_SYNC_ROUNDS,
16761            conflict: None,
16762            already_in: None,
16763        });
16764        runner.sync_to_base().await.unwrap();
16765        assert_eq!(runner.state.status, RunStatus::Blocked);
16766        (tmp, runner, head, task.id)
16767    }
16768
16769    #[cfg(unix)]
16770    #[tokio::test]
16771    async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16772        for routed in [false, true] {
16773            let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16774            let home = crate::run::home();
16775            let queue = crate::queue::Queue::open();
16776            let task = queue.get(&owner).unwrap();
16777            runner.state.task_instruction = Some(task.instruction.clone());
16778            runner.state.driver_exited = true;
16779            runner.state.save().unwrap();
16780            let questions = crate::ask::Questions::at(home.join("questions"));
16781            let mut q = crate::ask::Question::new(
16782                runner.state.id.clone(),
16783                "base-recovery".into(),
16784                "owner".into(),
16785                "Recover bounded base movement?".into(),
16786                String::new(),
16787                vec!["recover".into()],
16788            );
16789            q.actions.insert(
16790                "recover".into(),
16791                crate::ask::ChoiceAction::RecoverBase {
16792                    run: runner.state.id.clone(),
16793                    stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16794                    episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16795                },
16796            );
16797            q.routed_task = Some(owner.clone());
16798            q.answer(crate::ask::Answer::Choice("recover".into()))
16799                .unwrap();
16800            questions.put(&mut q).unwrap();
16801            crate::base_recovery_control::recover(
16802                &queue,
16803                &home,
16804                crate::base_recovery_control::Request {
16805                    run: runner.state.id.clone(),
16806                    head: head.clone(),
16807                    actor: "decisions".into(),
16808                    request: q.id.clone(),
16809                    decision: Some((owner.clone(), q.answered_at.unwrap())),
16810                    expected_stop: Some((
16811                        runner.state.base_sync_recovery.stop.clone().unwrap(),
16812                        runner.state.base_sync_recovery.episodes.last().unwrap().id,
16813                    )),
16814                },
16815            )
16816            .await
16817            .unwrap();
16818            if routed {
16819                questions
16820                    .update(&q.id, |q| {
16821                        q.routed_task = Some("another-owner".into());
16822                        Ok(())
16823                    })
16824                    .unwrap();
16825            } else {
16826                let mut task = queue.get(&owner).unwrap();
16827                task.hold_manual(Some("new owner decision".into()));
16828                // Keep the generation to exercise the separate owner-decision binding.
16829                task.status = crate::queue::TaskStatus::Queued;
16830                queue.put(&mut task).unwrap();
16831            }
16832            runner.state = RunState::load(&runner.state.id).unwrap();
16833            runner.sync_to_base().await.unwrap();
16834            let approval = runner
16835                .state
16836                .base_sync_recovery
16837                .authorizations
16838                .last()
16839                .unwrap();
16840            assert!(approval.consumed);
16841            assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
16842                "routing"
16843            } else {
16844                "owner decision"
16845            }));
16846            assert_eq!(
16847                runner.state.base_sync.as_ref().unwrap().attempts,
16848                BASE_SYNC_ROUNDS
16849            );
16850            assert_eq!(
16851                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16852                head
16853            );
16854        }
16855    }
16856
16857    #[cfg(unix)]
16858    #[tokio::test]
16859    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16860        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16861        if std::env::var_os(CHILD).is_none() {
16862            let home = tempfile::tempdir().unwrap();
16863            let output = std::process::Command::new(std::env::current_exe().unwrap())
16864                .args([
16865                    "--exact",
16866                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16867                    "--nocapture",
16868                ])
16869                .env(CHILD, "1")
16870                .env("MAGI_HOME", home.path())
16871                .env("MAGI_WORKER_HOME", home.path())
16872                .env("MAGI_WORKER_ID", "abcdef")
16873                .output()
16874                .unwrap();
16875            assert!(
16876                output.status.success()
16877                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16878                "{}\n{}",
16879                String::from_utf8_lossy(&output.stdout),
16880                String::from_utf8_lossy(&output.stderr)
16881            );
16882            return;
16883        }
16884        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16885        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16886        let home = crate::run::home();
16887        let queue = crate::queue::Queue::open();
16888        let mut owner = queue.get(&task).unwrap();
16889        owner.release_operator("test-release", None);
16890        owner.start(runner.state.id.clone());
16891        queue.put(&mut owner).unwrap();
16892        runner.state.driver_exited = true;
16893        runner.state.task_instruction = Some(owner.instruction.clone());
16894        runner.state.save().unwrap();
16895        let _claim = queue.claim(&task).unwrap();
16896        let reserve = |id: &str| {
16897            let directory = home.join("workers/jobs").join(id);
16898            std::fs::create_dir_all(&directory).unwrap();
16899            let job = crate::worker::Job {
16900                id: id.into(),
16901                sequence: 1,
16902                kind: "task".into(),
16903                key: format!("task:{task}"),
16904                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16905                generation: "fixture-generation".into(),
16906                version: None,
16907                status: crate::worker::Status::Running,
16908                result: None,
16909                pid: Some(std::process::id()),
16910                created_at: 0,
16911                finished_at: None,
16912            };
16913            std::fs::write(
16914                directory.join("job.json"),
16915                serde_json::to_vec(&job).unwrap(),
16916            )
16917            .unwrap();
16918            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16919        };
16920        let _own = reserve("abcdef");
16921        let other = reserve("fedcba");
16922        assert!(
16923            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16924                .await
16925                .is_err()
16926        );
16927        assert!(
16928            RunState::load(&runner.state.id)
16929                .unwrap()
16930                .base_sync_recovery
16931                .authorizations
16932                .is_empty()
16933        );
16934        drop(other);
16935        assert!(crate::daemon::is_working_on(
16936            &home,
16937            &runner.state.id,
16938            Timestamp::now()
16939        ));
16940        assert!(
16941            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16942        );
16943        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16944            .await
16945            .unwrap();
16946        assert!(
16947            RunState::load(&runner.state.id)
16948                .unwrap()
16949                .base_recovery_pending()
16950        );
16951    }
16952
16953    #[cfg(unix)]
16954    #[tokio::test]
16955    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16956        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16957        let home = crate::run::home();
16958        runner.state.driver_pid = Some(std::process::id());
16959        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16960        runner.state.driver_exited = false;
16961        runner.state.begin_pass();
16962        runner.state.save().unwrap();
16963        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16964        driver.record(&runner.state).unwrap();
16965        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16966        // Preserve the durable claim as if the process died before Drop saved.
16967        drop(driver);
16968        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16969        assert_eq!(
16970            runner.state.liveness_under(&home, false),
16971            crate::run::Liveness::Dead
16972        );
16973        let queue = crate::queue::Queue::open();
16974        let _claim = queue.claim(&task).unwrap();
16975        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16976            .await
16977            .unwrap();
16978        let disk = RunState::load(&runner.state.id).unwrap();
16979        assert!(!disk.driver_exited);
16980        assert!(disk.base_recovery_pending());
16981    }
16982
16983    #[cfg(unix)]
16984    #[tokio::test]
16985    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16986        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16987        if std::env::var_os(CHILD).is_none() {
16988            use std::os::unix::fs::PermissionsExt;
16989            let tmp = tempfile::tempdir().unwrap();
16990            let gh = tmp.path().join("gh");
16991            std::fs::write(&gh, r#"#!/bin/sh
16992case "$1 $2" in
16993  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16994  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16995  *) exit 2 ;;
16996esac
16997"#).unwrap();
16998            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16999            let mut paths = vec![tmp.path().to_path_buf()];
17000            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17001            let output = std::process::Command::new(std::env::current_exe().unwrap())
17002                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17003                .env(CHILD, "1")
17004                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17005                .env("PATH", std::env::join_paths(paths).unwrap())
17006                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17007                .output().unwrap();
17008            assert!(
17009                output.status.success()
17010                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17011                "{}\n{}",
17012                String::from_utf8_lossy(&output.stdout),
17013                String::from_utf8_lossy(&output.stderr)
17014            );
17015            return;
17016        }
17017        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17018        let repo = runner.state.repo.clone();
17019        git::git(&repo, &["checkout", "main"]).await.unwrap();
17020        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17021        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17022        git::git(&repo, &["checkout", "fix/retained"])
17023            .await
17024            .unwrap();
17025        runner.state.pr = Some(crate::run::PrRecord {
17026            url: "https://example.test/project/pull/1".into(),
17027            number: 1,
17028            state: "open".into(),
17029            checks: "unknown".into(),
17030            round: 0,
17031            rounds: 1,
17032            red_at_merge: Vec::new(),
17033        });
17034        runner.sync_to_base().await.unwrap();
17035        assert_eq!(runner.state.status, RunStatus::Blocked);
17036        assert_eq!(
17037            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17038            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17039        );
17040        assert!(runner.state.base_sync_closure_pending());
17041        assert!(crate::handover::can_resume(&runner.state));
17042        let queue = crate::queue::Queue::open();
17043        assert!(!crate::base_sync_recovery::held_task(
17044            &queue,
17045            &queue.get(&task).unwrap()
17046        ));
17047        assert!(runner.state.authorize_base_sync(&task).is_err());
17048        // Legacy closure stops have no typed stop but must retain the checkpoint.
17049        runner.state.base_sync_recovery.stop = None;
17050        runner.state.status = RunStatus::Voting;
17051        runner.state.gate_ran = true;
17052        runner.state.config.graph.review_rounds = 0;
17053        runner.state.save().unwrap();
17054        assert!(crate::handover::can_resume(&runner.state));
17055        runner.state = RunState::load(&runner.state.id).unwrap();
17056        runner.execute_graph().await.unwrap();
17057        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17058        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17059        assert_eq!(
17060            runner.state.base_sync.as_ref().unwrap().attempts,
17061            BASE_SYNC_ROUNDS
17062        );
17063        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17064        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17065        assert!(runner.state.reviews.is_empty());
17066        assert!(runner.state.gate.is_empty());
17067    }
17068
17069    #[cfg(unix)]
17070    #[tokio::test]
17071    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17072        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17073        runner.state.driver_exited = true;
17074        runner.state.save().unwrap();
17075        let home = crate::run::home();
17076        let queue = crate::queue::Queue::open();
17077        let _claim = queue.claim(&task).unwrap();
17078        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17079        assert!(
17080            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17081                .await
17082                .is_err()
17083        );
17084        drop(lease);
17085        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17086        assert!(
17087            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17088                .await
17089                .is_err()
17090        );
17091        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17092        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17093            .await
17094            .unwrap();
17095        let disk = RunState::load(&runner.state.id).unwrap();
17096        assert!(disk.base_recovery_pending());
17097        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17098        assert!(
17099            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17100                .await
17101                .is_err()
17102        );
17103        assert_eq!(
17104            RunState::load(&runner.state.id)
17105                .unwrap()
17106                .base_sync_recovery
17107                .authorizations
17108                .len(),
17109            1
17110        );
17111        let mut owner = queue.get(&task).unwrap();
17112        owner.runs.push("20260101-000000-ffff".into());
17113        queue.put(&mut owner).unwrap();
17114        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17115            .await
17116            .unwrap_err();
17117        assert!(error.to_string().contains("run task ownership changed"));
17118    }
17119
17120    #[cfg(unix)]
17121    #[tokio::test]
17122    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17123        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17124        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17125        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17126        legacy["status"] = serde_json::json!("voting");
17127        runner.state = serde_json::from_value(legacy).unwrap();
17128        runner.state.save().unwrap();
17129        for _ in 0..3 {
17130            runner.execute_graph().await.unwrap();
17131            assert_eq!(runner.state.status, RunStatus::Blocked);
17132            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17133            assert!(!crate::handover::can_resume(&runner.state));
17134            assert_eq!(
17135                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17136                head
17137            );
17138            assert!(runner.state.reviews.is_empty());
17139            assert!(!runner.state.gate_ran);
17140        }
17141        let notice = crate::notices::run_ended(&runner.state).unwrap();
17142        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17143        runner.state.begin_pass();
17144        runner.state.status = RunStatus::Voting;
17145        runner.deliberate().await.unwrap();
17146        runner.vote().await.unwrap();
17147        runner.finish_after_tally().await.unwrap();
17148        assert_eq!(runner.state.status, RunStatus::Blocked);
17149        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17150        assert_eq!(
17151            crate::notices::run_ended(&runner.state).unwrap().since,
17152            notice.since
17153        );
17154    }
17155
17156    #[cfg(unix)]
17157    #[tokio::test]
17158    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17159        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17160        for detail in [
17161            "fix/retained conflicts with origin/main and did not rebase: conflict",
17162            "rebased fix/retained locally but origin refused the push",
17163            "unknown failure",
17164        ] {
17165            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17166            runner.state.record_base_sync_stop(&head);
17167            assert!(runner.state.authorize_base_sync(&task).is_err());
17168            runner.state.status = RunStatus::Voting;
17169            runner.execute_graph().await.unwrap();
17170            assert_eq!(runner.state.status, RunStatus::Blocked);
17171            assert!(!crate::handover::can_resume(&runner.state));
17172            assert_eq!(
17173                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17174                head
17175            );
17176        }
17177    }
17178
17179    #[cfg(unix)]
17180    #[tokio::test]
17181    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17182        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17183        runner.state.authorize_base_sync(&task).unwrap();
17184        assert!(runner.state.consume_base_recovery(&head));
17185        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17186        runner.state.reserve_base_operation(&head, &tip);
17187        runner.state.save().unwrap();
17188        runner.state = RunState::load(&runner.state.id).unwrap();
17189        assert!(!runner.state.consume_base_recovery(&head));
17190        runner.sync_to_base().await.unwrap();
17191        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17192        assert_eq!(runner.state.base_episode_used(), 1);
17193        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17194        assert!(
17195            runner
17196                .state
17197                .base_sync_recovery
17198                .operations
17199                .last()
17200                .unwrap()
17201                .finished
17202        );
17203        assert!(!runner.state.base_sync_blocked());
17204        runner.sync_to_base().await.unwrap();
17205        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17206    }
17207
17208    #[cfg(unix)]
17209    #[tokio::test]
17210    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17211        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17212        // Both historical diagnostic spellings retain an exhaustion identity.
17213        let original = runner
17214            .state
17215            .base_sync
17216            .as_ref()
17217            .unwrap()
17218            .conflict
17219            .clone()
17220            .unwrap();
17221        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17222            original
17223                .replace("synchronization(s)", "rebase(s)")
17224                .replace("synchronizing", "rebasing"),
17225        );
17226        runner.state.base_sync_recovery = Default::default();
17227        runner.state.record_base_sync_stop(&head);
17228        assert_eq!(
17229            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17230            crate::base_sync_recovery::StopKind::Exhausted
17231        );
17232        runner.state.authorize_base_sync(&task).unwrap();
17233        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17234        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17235            .as_object_mut()
17236            .unwrap();
17237        auth.remove("interval");
17238        auth.remove("scope");
17239        runner.state = serde_json::from_value(legacy).unwrap();
17240        assert!(runner.state.consume_base_recovery(&head));
17241        assert_eq!(runner.state.base_episode_used(), 3);
17242        runner.sync_to_base().await.unwrap();
17243        assert_eq!(runner.state.base_episode_used(), 4);
17244        runner.state.resource_block = Some(crate::run::ResourceBlock {
17245            node: "gate".into(),
17246            kind: "cache".into(),
17247            detail: "legacy wait".into(),
17248            at: Timestamp::now(),
17249            cause: crate::run::ResourceBlockCause::Unrecorded,
17250        });
17251        runner.state.save().unwrap();
17252        assert!(
17253            runner
17254                .state
17255                .base_sync_recovery
17256                .episodes
17257                .last()
17258                .unwrap()
17259                .boundary
17260                .is_none()
17261        );
17262        runner.state.resource_block = None;
17263        advance_base_fixture(&runner).await;
17264        runner.sync_to_base().await.unwrap();
17265        assert!(runner.state.base_sync_blocked());
17266        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17267    }
17268
17269    #[cfg(unix)]
17270    async fn advance_base_fixture(runner: &Runner) {
17271        let repo = &runner.state.repo;
17272        for args in [
17273            vec!["checkout", "main"],
17274            vec!["commit", "--allow-empty", "-m", "base advanced"],
17275            vec!["push", "origin", "main"],
17276            vec!["checkout", "fix/retained"],
17277        ] {
17278            git::git(repo, &args).await.unwrap();
17279        }
17280    }
17281
17282    #[cfg(unix)]
17283    fn fresh_base_interval_fixture(runner: &mut Runner) {
17284        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17285        runner.state.base_sync.as_mut().unwrap().conflict = None;
17286        runner.state.base_sync_recovery = Default::default();
17287        runner.state.status = RunStatus::Voting;
17288    }
17289
17290    #[cfg(unix)]
17291    #[tokio::test]
17292    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17293        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17294        fresh_base_interval_fixture(&mut runner);
17295        runner.state.begin_pass();
17296        runner.sync_to_base().await.unwrap();
17297        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17298        runner.state.resource_block = Some(crate::run::ResourceBlock {
17299            node: "gate".into(),
17300            kind: "build cache".into(),
17301            detail: "busy".into(),
17302            at: Timestamp::now(),
17303            cause: crate::run::ResourceBlockCause::Timeout {
17304                waited_secs: 1,
17305                limit_secs: 1,
17306            },
17307        });
17308        runner.state.save().unwrap();
17309        let queue = crate::queue::Queue::open();
17310        let mut owner = queue.get(&task).unwrap();
17311        runner.state.task_instruction = Some(owner.instruction.clone());
17312        owner.release_operator("fixture-owner", None);
17313        queue.put(&mut owner).unwrap();
17314        let approval = owner.resumption.clone().unwrap();
17315        runner.state.resource_block = None;
17316        runner.state.save().unwrap();
17317        runner.state = RunState::load(&runner.state.id).unwrap();
17318        advance_base_fixture(&runner).await;
17319        runner.sync_to_base().await.unwrap();
17320        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17321        assert_eq!(runner.state.base_episode_used(), 1);
17322        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17323        assert!(
17324            runner.state.base_sync_recovery.episodes[0]
17325                .boundary
17326                .as_ref()
17327                .unwrap()
17328                .consumed
17329        );
17330        assert!(
17331            !runner
17332                .state
17333                .renew_base_episode(&task, &approval, &head, "base")
17334        );
17335        for _ in 0..3 {
17336            owner.release_operator("fixture-owner", None);
17337            queue.put(&mut owner).unwrap();
17338            runner.state = RunState::load(&runner.state.id).unwrap();
17339            runner.state.begin_pass();
17340            runner.sync_to_base().await.unwrap();
17341            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17342            assert_eq!(runner.state.base_episode_used(), 1);
17343        }
17344        runner.state.resource_block = Some(crate::run::ResourceBlock {
17345            node: "gate".into(),
17346            kind: "disk pressure".into(),
17347            detail: "free space required".into(),
17348            at: Timestamp::now(),
17349            cause: crate::run::ResourceBlockCause::Other,
17350        });
17351        runner.state.end_pass_parked(ParkCause::Interrupt {
17352            reason: "disk pressure".into(),
17353        });
17354        runner.state.save().unwrap();
17355        owner.release_operator("fixture-owner-after-disk", None);
17356        queue.put(&mut owner).unwrap();
17357        runner.state.resource_block = None;
17358        advance_base_fixture(&runner).await;
17359        runner.sync_to_base().await.unwrap();
17360        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17361        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17362        assert_eq!(runner.state.base_episode_used(), 1);
17363        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17364    }
17365
17366    #[cfg(unix)]
17367    #[tokio::test]
17368    async fn base_recovery_continuous_advancement_stops_at_four() {
17369        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17370        fresh_base_interval_fixture(&mut runner);
17371        for spent in 1..=BASE_SYNC_ROUNDS {
17372            runner.sync_to_base().await.unwrap();
17373            assert!(!runner.state.base_sync_blocked());
17374            assert_eq!(runner.state.base_episode_used(), spent);
17375            advance_base_fixture(&runner).await;
17376        }
17377        runner.sync_to_base().await.unwrap();
17378        assert_eq!(
17379            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17380            crate::base_sync_recovery::StopKind::Exhausted
17381        );
17382        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17383        assert!(stop.unverified_sync);
17384        let action = runner.state.base_sync_next_action();
17385        assert!(action.contains("never reached its gate"));
17386        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17387        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17388        runner.state = RunState::load(&runner.state.id).unwrap();
17389        runner.sync_to_base().await.unwrap();
17390        assert_eq!(runner.state.base_episode_used(), 4);
17391        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17392    }
17393
17394    #[cfg(unix)]
17395    #[tokio::test]
17396    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17397        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17398        runner.state.ensure_base_episode();
17399        // Verification progress alone is not evidence of a successful sync.
17400        runner.state.record_base_progress(&head);
17401        runner.state.base_sync_recovery.stop = None;
17402        runner.state.record_base_sync_stop(&head);
17403        assert!(
17404            !runner
17405                .state
17406                .base_sync_recovery
17407                .stop
17408                .as_ref()
17409                .unwrap()
17410                .unverified_sync
17411        );
17412
17413        runner.state.finish_base_operation(Some(&head));
17414        // A gate for another head cannot verify this synchronization.
17415        runner.state.gate_ran = true;
17416        runner.state.gate_head = Some("older-head".into());
17417        runner.state.base_sync_recovery.stop = None;
17418        runner.state.record_base_sync_stop(&head);
17419        runner.state.save().unwrap();
17420        runner.state = RunState::load(&runner.state.id).unwrap();
17421        assert!(
17422            runner
17423                .state
17424                .base_sync_recovery
17425                .stop
17426                .as_ref()
17427                .unwrap()
17428                .unverified_sync
17429        );
17430
17431        runner.state.gate_head = Some(head.clone());
17432        runner.state.base_sync_recovery.stop = None;
17433        runner.state.record_base_sync_stop(&head);
17434        assert!(
17435            !runner
17436                .state
17437                .base_sync_recovery
17438                .stop
17439                .as_ref()
17440                .unwrap()
17441                .unverified_sync
17442        );
17443        assert!(
17444            !runner
17445                .state
17446                .base_sync_next_action()
17447                .contains("never reached its gate")
17448        );
17449
17450        // Historical records cannot establish the missing synchronization evidence.
17451        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17452        legacy["base_sync_recovery"]["stop"]
17453            .as_object_mut()
17454            .unwrap()
17455            .remove("unverified_sync");
17456        legacy["base_sync_recovery"]["episodes"][0]
17457            .as_object_mut()
17458            .unwrap()
17459            .remove("last_sync_head");
17460        runner.state = serde_json::from_value(legacy).unwrap();
17461        assert!(
17462            !runner
17463                .state
17464                .base_sync_recovery
17465                .stop
17466                .as_ref()
17467                .unwrap()
17468                .unverified_sync
17469        );
17470        assert!(
17471            runner.state.base_sync_recovery.episodes[0]
17472                .last_sync_head
17473                .is_none()
17474        );
17475    }
17476
17477    #[cfg(unix)]
17478    #[tokio::test]
17479    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17480        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17481        runner.state.authorize_base_sync(&task).unwrap();
17482        assert!(
17483            runner
17484                .state
17485                .base_sync_next_action()
17486                .contains("Approval recorded")
17487        );
17488        assert!(
17489            runner
17490                .state
17491                .base_sync_next_action()
17492                .contains("4/4 in episode 0, 4 lifetime")
17493        );
17494        assert!(runner.state.consume_base_recovery(&head));
17495        assert!(
17496            runner
17497                .state
17498                .base_sync_next_action()
17499                .contains("0/4 in episode 1, 4 lifetime")
17500        );
17501    }
17502
17503    #[cfg(unix)]
17504    #[tokio::test]
17505    async fn base_recovery_changed_head_invalidates_without_spending() {
17506        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17507        runner.state.authorize_base_sync(&task).unwrap();
17508        git::git(
17509            &runner.state.repo,
17510            &["commit", "--allow-empty", "-m", "external head change"],
17511        )
17512        .await
17513        .unwrap();
17514        runner.sync_to_base().await.unwrap();
17515        let a = runner
17516            .state
17517            .base_sync_recovery
17518            .authorizations
17519            .last()
17520            .unwrap();
17521        assert!(a.consumed);
17522        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17523        runner.state = RunState::load(&runner.state.id).unwrap();
17524        assert!(
17525            runner
17526                .state
17527                .base_sync_next_action()
17528                .contains("Approval invalidated")
17529        );
17530        assert!(!runner.state.base_recovery_pending());
17531        assert!(runner.state.base_sync_blocked());
17532        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17533    }
17534
17535    #[cfg(unix)]
17536    #[tokio::test]
17537    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17538        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17539        runner.state.driver_exited = true;
17540        runner.state.save().unwrap();
17541        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17542            .await
17543            .unwrap();
17544        runner.state = RunState::load(&runner.state.id).unwrap();
17545        advance_base_fixture(&runner).await;
17546        runner.sync_to_base().await.unwrap();
17547        let a = runner
17548            .state
17549            .base_sync_recovery
17550            .authorizations
17551            .last()
17552            .unwrap();
17553        assert!(
17554            a.consumed
17555                && a.invalidated
17556                    .as_deref()
17557                    .unwrap()
17558                    .contains("fetched base changed")
17559        );
17560        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17561        assert_eq!(
17562            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17563            head
17564        );
17565        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17566            .await
17567            .unwrap();
17568        runner.state = RunState::load(&runner.state.id).unwrap();
17569        runner.sync_to_base().await.unwrap();
17570        assert!(!runner.state.base_sync_blocked());
17571        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17572    }
17573
17574    #[cfg(unix)]
17575    #[tokio::test]
17576    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17577        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17578        runner.state.authorize_base_sync(&task).unwrap();
17579        assert!(runner.state.consume_base_recovery(&head));
17580        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17581        runner.state.reserve_base_operation(&head, &tip);
17582        runner.state.save().unwrap();
17583        let scratch = runner.state.dir().join("base-sync");
17584        assert_eq!(
17585            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17586                .await
17587                .unwrap(),
17588            crate::rebase::Rebased::Applied
17589        );
17590        runner.state = RunState::load(&runner.state.id).unwrap();
17591        runner.sync_to_base().await.unwrap();
17592        assert!(
17593            runner
17594                .state
17595                .base_sync_recovery
17596                .operations
17597                .last()
17598                .unwrap()
17599                .finished
17600        );
17601        assert_eq!(runner.state.base_episode_used(), 1);
17602        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17603        assert!(!runner.state.base_sync_blocked());
17604    }
17605
17606    #[cfg(unix)]
17607    #[tokio::test]
17608    async fn base_recovery_crash_preserves_untracked_files() {
17609        for ignored in [false, true] {
17610            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17611            runner.state.authorize_base_sync(&task).unwrap();
17612            assert!(runner.state.consume_base_recovery(&head));
17613            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17614            runner.state.reserve_base_operation(&head, &tip);
17615            runner.state.save().unwrap();
17616            let scratch = runner.state.dir().join("base-sync");
17617            assert_eq!(
17618                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17619                    .await
17620                    .unwrap(),
17621                crate::rebase::Rebased::Applied
17622            );
17623            let repo = &runner.state.repo;
17624            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17625            assert_ne!(result, head);
17626            if ignored {
17627                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17628                    .await
17629                    .unwrap();
17630                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17631            }
17632            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17633            assert!(
17634                git::git_raw(repo, &["diff", "--quiet", &head])
17635                    .await
17636                    .unwrap()
17637                    .ok()
17638            );
17639            // Resume after the branch moved, before its checkout was reset.
17640            runner.state = RunState::load(&runner.state.id).unwrap();
17641            runner.sync_to_base().await.unwrap();
17642            assert!(runner.state.base_sync_blocked());
17643            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17644            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17645            assert_eq!(
17646                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17647                result
17648            );
17649            assert_eq!(
17650                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17651                "retained work\n"
17652            );
17653            assert!(
17654                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17655                    .await
17656                    .unwrap()
17657                    .ok()
17658            );
17659        }
17660    }
17661
17662    #[cfg(unix)]
17663    #[tokio::test]
17664    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17665        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17666        runner.state.authorize_base_sync(&task).unwrap();
17667        assert!(runner.state.consume_base_recovery(&head));
17668        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17669        runner.state.reserve_base_operation(&head, &tip);
17670        runner.state.save().unwrap();
17671        let scratch = runner.state.dir().join("base-sync");
17672        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17673            .await
17674            .unwrap();
17675        git::sync_to_head(&runner.state.repo).await.unwrap();
17676        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17677        git::git(&runner.state.repo, &["add", "foreign"])
17678            .await
17679            .unwrap();
17680        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17681            .await
17682            .unwrap();
17683        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17684        runner.state = RunState::load(&runner.state.id).unwrap();
17685        runner.sync_to_base().await.unwrap();
17686        assert!(runner.state.base_sync_blocked());
17687        assert!(
17688            !runner
17689                .state
17690                .base_sync_recovery
17691                .operations
17692                .last()
17693                .unwrap()
17694                .finished
17695        );
17696        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17697        assert_eq!(
17698            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17699            foreign_head
17700        );
17701    }
17702
17703    #[cfg(unix)]
17704    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17705    {
17706        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17707        fresh_base_interval_fixture(&mut runner);
17708        let repo = &runner.state.repo;
17709        git::git(repo, &["checkout", "main"]).await.unwrap();
17710        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17711        git::git(repo, &["add", "retained"]).await.unwrap();
17712        git::git(repo, &["commit", "-m", "upstream retained change"])
17713            .await
17714            .unwrap();
17715        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17716        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17717        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17718        let scratch = runner.state.dir().join("base-sync");
17719        assert!(matches!(
17720            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17721                .await
17722                .unwrap(),
17723            git::RebaseStart::Conflicted(_)
17724        ));
17725        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17726            agent: "fixture".into(),
17727            paths: vec!["retained".into()],
17728            from: Some(head.clone()),
17729            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17730            index_outside: None,
17731            finished: false,
17732            error: None,
17733            launcher: None,
17734            infra: None,
17735            outage: None,
17736        });
17737        (tmp, runner, head, tip, scratch)
17738    }
17739
17740    #[cfg(unix)]
17741    #[tokio::test]
17742    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17743        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17744        // A resource stop from before reservations existed, with resolved
17745        // contents left for the launcher's ordinary staging continuation.
17746        std::fs::write(
17747            scratch.join("retained"),
17748            "upstream and retained resolution\n",
17749        )
17750        .unwrap();
17751        runner.state.resource_block = Some(crate::run::ResourceBlock {
17752            node: "rebase".into(),
17753            kind: "repository coordination".into(),
17754            detail: "busy".into(),
17755            at: Timestamp::now(),
17756            cause: crate::run::ResourceBlockCause::Other,
17757        });
17758        runner.state.save().unwrap();
17759        // Advance the remote without moving either checkout participating
17760        // in the standing rebase.
17761        let newer = git::git(
17762            &runner.state.repo,
17763            &[
17764                "commit-tree",
17765                "main^{tree}",
17766                "-p",
17767                "main",
17768                "-m",
17769                "base advanced during wait",
17770            ],
17771        )
17772        .await
17773        .unwrap();
17774        git::git(
17775            &runner.state.repo,
17776            &[
17777                "push",
17778                "origin",
17779                &format!("{}:refs/heads/main", newer.trim()),
17780            ],
17781        )
17782        .await
17783        .unwrap();
17784        runner.state = RunState::load(&runner.state.id).unwrap();
17785        runner.state.resource_block = None;
17786        runner.sync_to_base().await.unwrap();
17787        assert!(!runner.state.base_sync_blocked());
17788        let op = &runner.state.base_sync_recovery.operations[0];
17789        assert_eq!(op.head, head);
17790        assert_eq!(op.tip, tip);
17791        assert!(op.finished);
17792        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17793        assert_eq!(runner.state.rebase_fixes.len(), 1);
17794        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17795            .await
17796            .unwrap();
17797        assert_ne!(latest, tip);
17798        runner.state = RunState::load(&runner.state.id).unwrap();
17799        runner.sync_to_base().await.unwrap();
17800        assert!(!runner.state.base_sync_blocked());
17801        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17802        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17803        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17804    }
17805
17806    #[cfg(unix)]
17807    #[tokio::test]
17808    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17809        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17810        runner.state.reserve_base_operation(&head, &tip);
17811        runner.state.save().unwrap();
17812        std::fs::write(
17813            scratch.join("retained"),
17814            "upstream and retained resolution\n",
17815        )
17816        .unwrap();
17817        git::git(&scratch, &["add", "retained"]).await.unwrap();
17818        git::git(
17819            &scratch,
17820            &["-c", "core.editor=true", "rebase", "--continue"],
17821        )
17822        .await
17823        .unwrap();
17824        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17825        assert!(
17826            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17827                .await
17828                .unwrap()
17829                .is_empty()
17830        );
17831        assert_eq!(
17832            crate::rebase::finish(
17833                &mut runner.state,
17834                &scratch,
17835                "fix/retained",
17836                &head,
17837                &tip,
17838                &["retained".into()],
17839                ""
17840            )
17841            .await
17842            .unwrap(),
17843            crate::rebase::Rebased::Applied
17844        );
17845        assert!(!scratch.exists());
17846        // Simulate losing the caller before finish_base_operation/save.
17847        runner.state = RunState::load(&runner.state.id).unwrap();
17848        let op = &runner.state.base_sync_recovery.operations[0];
17849        assert!(!op.finished);
17850        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17851        runner.sync_to_base().await.unwrap();
17852        assert!(!runner.state.base_sync_blocked());
17853        assert!(runner.state.base_sync_recovery.operations[0].finished);
17854        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17855        assert_eq!(runner.state.rebase_fixes.len(), 1);
17856        let winner = runner.state.winner().unwrap().clone();
17857        let wait = runner.repository_wait();
17858        assert!(
17859            verify_current_head(&mut runner.state, &winner, &wait)
17860                .await
17861                .unwrap()
17862        );
17863        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17864    }
17865
17866    #[cfg(unix)]
17867    #[tokio::test]
17868    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17869        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17870            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17871            runner.state.reserve_base_operation(&head, &tip);
17872            runner.state.save().unwrap();
17873            std::fs::write(
17874                scratch.join("retained"),
17875                "upstream and retained resolution\n",
17876            )
17877            .unwrap();
17878            git::git(&scratch, &["add", "retained"]).await.unwrap();
17879            git::git(
17880                &scratch,
17881                &["-c", "core.editor=true", "rebase", "--continue"],
17882            )
17883            .await
17884            .unwrap();
17885            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17886            assert!(!git::rebase_in_progress(&scratch).await);
17887            assert!(
17888                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17889                    .await
17890                    .unwrap()
17891                    .is_empty()
17892            );
17893            match damage {
17894                Some("record") => {
17895                    let path =
17896                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17897                            .await
17898                            .unwrap();
17899                    std::fs::write(scratch.join(path), "").unwrap();
17900                }
17901                Some("foreign") => {
17902                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17903                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17904                    git::git(&scratch, &["commit", "-m", "foreign work"])
17905                        .await
17906                        .unwrap();
17907                }
17908                Some("dirty") => {
17909                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17910                }
17911                _ => {}
17912            }
17913            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17914            // Crash immediately after Git continuation, before finish runs.
17915            runner.state = RunState::load(&runner.state.id).unwrap();
17916            assert!(
17917                runner.state.base_sync_recovery.operations[0]
17918                    .validated_head
17919                    .is_none()
17920            );
17921            runner.sync_to_base().await.unwrap();
17922            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17923            assert_eq!(runner.state.base_episode_used(), 1);
17924            assert_eq!(runner.state.rebase_fixes.len(), 1);
17925            assert_eq!(
17926                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17927                retained_head
17928            );
17929            if damage.is_some() {
17930                assert!(runner.state.base_sync_blocked());
17931                assert!(scratch.exists());
17932                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17933                assert!(
17934                    runner.state.base_sync_recovery.operations[0]
17935                        .validated_head
17936                        .is_none()
17937                );
17938            } else {
17939                assert!(!runner.state.base_sync_blocked());
17940                assert!(!scratch.exists());
17941                assert!(runner.state.base_sync_recovery.operations[0].finished);
17942                assert_eq!(
17943                    runner.state.base_sync_recovery.operations[0]
17944                        .validated_head
17945                        .as_deref(),
17946                    Some(result.as_str())
17947                );
17948                let winner = runner.state.winner().unwrap().clone();
17949                let wait = runner.repository_wait();
17950                assert!(
17951                    verify_current_head(&mut runner.state, &winner, &wait)
17952                        .await
17953                        .unwrap()
17954                );
17955                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17956                runner.state = RunState::load(&runner.state.id).unwrap();
17957                runner.sync_to_base().await.unwrap();
17958                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17959            }
17960        }
17961    }
17962
17963    #[cfg(unix)]
17964    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17965        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17966        // This is an ordinary gate stop, not an authorized base-budget recovery.
17967        runner.state.prep_complete = Some(true);
17968        runner.state.base_sync = None;
17969        runner.state.base_sync_recovery = Default::default();
17970        runner.state.config.graph.gate_fix_rounds = 1;
17971        let mut review = review_round(true, 0, 1, 1, false, true);
17972        review.head = head.clone();
17973        review.verified_head = Some(head.clone());
17974        runner.state.reviews = vec![review];
17975        runner.state.gate = vec![CommandOutcome {
17976            command: "old gate".into(),
17977            code: Some(1),
17978            output_tail: "old failure".into(),
17979            duration_ms: 0,
17980            resource_blocked: false,
17981        }];
17982        runner.state.gate_head = Some(head.clone());
17983        runner.state.gate_ran = true;
17984        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17985            agent: "fixture".into(),
17986            failed: runner.state.gate.clone(),
17987            notes: "spent".into(),
17988            committed: false,
17989            error: None,
17990        }];
17991        runner.state.land_approval = Some(crate::run::LandApproval {
17992            question: "old approval".into(),
17993            head: head.clone(),
17994        });
17995        runner.state.status = RunStatus::Blocked;
17996        (tmp, runner, head)
17997    }
17998
17999    #[cfg(unix)]
18000    #[tokio::test]
18001    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18002        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18003            let (tmp, mut runner, old) = resumed_gate_fixture().await;
18004            let phases = tmp.path().join("phases");
18005            runner.state.config.verify.e2e = vec![format!(
18006                "echo e2e >> '{}'; {}",
18007                phases.display(),
18008                if e2e_ok {
18009                    "true"
18010                } else {
18011                    "echo new-e2e-failure; false"
18012                }
18013            )];
18014            runner.state.config.verify.gate = vec![format!(
18015                "echo gate >> '{}'; {}",
18016                phases.display(),
18017                if gate_ok {
18018                    "true"
18019                } else {
18020                    "echo new-gate-failure; false"
18021                }
18022            )];
18023            runner.state.save().unwrap();
18024            runner.state = RunState::load(&runner.state.id).unwrap();
18025            runner.sync_to_base().await.unwrap();
18026            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18027            assert_ne!(head, old);
18028            // Reentry after Git updated but before verification was invalidated.
18029            runner.state = RunState::load(&runner.state.id).unwrap();
18030            runner.execute_graph().await.unwrap();
18031            assert_eq!(runner.state.gate_fixes.len(), 1);
18032            assert_eq!(runner.state.reviews.len(), 1);
18033            assert_eq!(runner.state.reviews[0].head, old);
18034            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18035            assert_eq!(runner.state.verification_history.len(), 1);
18036            let history = &runner.state.verification_history[0];
18037            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18038            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18039            assert!(history.archived_at.is_some());
18040            assert_eq!(history.fixes_spent, Some(1));
18041            assert_eq!(history.gate[0].output_tail, "old failure");
18042            let archive_events: Vec<_> = runner
18043                .state
18044                .events
18045                .iter()
18046                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18047                .collect();
18048            assert_eq!(archive_events.len(), 1);
18049            assert!(archive_events[0].message.contains(&short(&old)));
18050            assert!(archive_events[0].message.contains(&short(&head)));
18051            assert!(
18052                archive_events[0]
18053                    .message
18054                    .contains("1 gate-fix round(s) already spent stay spent")
18055            );
18056            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18057            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18058            if e2e_ok {
18059                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18060                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18061            } else {
18062                assert!(!runner.state.gate_ran);
18063            }
18064            assert_eq!(
18065                runner.state.status,
18066                if e2e_ok && gate_ok {
18067                    RunStatus::Ready
18068                } else {
18069                    RunStatus::Blocked
18070                }
18071            );
18072            if !e2e_ok || !gate_ok {
18073                let cause = runner.state.verification_blocking_cause().unwrap();
18074                assert!(cause.contains(&head));
18075                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18076                // A later successful progress event must not become the cause.
18077                runner.state.event("land", "rebased successfully");
18078                let notice = crate::notices::run_ended(&runner.state).unwrap();
18079                assert!(!format!("{notice:?}").contains("rebased successfully"));
18080                assert!(runner.state.merge.is_none());
18081            }
18082            let before = std::fs::read_to_string(&phases).unwrap();
18083            runner.state.save().unwrap();
18084            runner.state = RunState::load(&runner.state.id).unwrap();
18085            runner.review_loop().await.unwrap();
18086            runner.gate().await.unwrap();
18087            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18088            assert_eq!(runner.state.verification_history.len(), 1);
18089            let history = &runner.state.verification_history[0];
18090            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18091            assert!(history.archived_at.is_some());
18092            assert_eq!(history.fixes_spent, Some(1));
18093        }
18094    }
18095
18096    #[cfg(unix)]
18097    #[tokio::test]
18098    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18099        let (tmp, mut runner, head) = resumed_gate_fixture().await;
18100        let marker = tmp.path().join("gate-ran");
18101        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18102        runner.review_loop().await.unwrap();
18103        runner.gate().await.unwrap();
18104        assert_eq!(runner.state.status, RunStatus::Blocked);
18105        assert!(!marker.exists());
18106        assert!(runner.state.verification_history.is_empty());
18107        runner.state.gate_head = None;
18108        runner.gate().await.unwrap();
18109        assert!(marker.exists());
18110        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18111        assert_eq!(runner.state.verification_history.len(), 1);
18112        assert_eq!(runner.state.gate_fixes.len(), 1);
18113    }
18114
18115    #[cfg(unix)]
18116    #[tokio::test]
18117    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18118        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18119        runner.sync_to_base().await.unwrap();
18120        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18121        runner.state.reconcile_verification_head(&head);
18122        // Both crash-before-command and resource wait are unresolved evidence.
18123        for outcomes in [
18124            Vec::new(),
18125            vec![CommandOutcome {
18126                command: "true".into(),
18127                code: None,
18128                output_tail: "cache unavailable".into(),
18129                duration_ms: 0,
18130                resource_blocked: true,
18131            }],
18132        ] {
18133            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18134                head: head.clone(),
18135                passed: false,
18136                outcomes,
18137            });
18138            runner.state.gate_ran = false;
18139            runner.state.gate.clear();
18140            runner.state.save().unwrap();
18141            runner.state = RunState::load(&runner.state.id).unwrap();
18142            runner.merge().await.unwrap();
18143            assert!(runner.state.merge.is_none());
18144            runner.review_loop().await.unwrap();
18145            runner.gate().await.unwrap();
18146            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18147            assert!(runner.state.gate_allows_progress());
18148            assert_eq!(runner.state.gate_fixes.len(), 1);
18149            assert!(runner.state.merge.is_none());
18150        }
18151        // Moving the winner after a successful gate cannot publish an unchecked SHA.
18152        git::git(
18153            &runner.state.repo,
18154            &["commit", "--allow-empty", "-m", "external update"],
18155        )
18156        .await
18157        .unwrap();
18158        runner.merge().await.unwrap();
18159        assert!(runner.state.merge.is_none());
18160        assert_eq!(runner.state.status, RunStatus::Blocked);
18161    }
18162
18163    #[cfg(unix)]
18164    #[tokio::test]
18165    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18166        for waiting_on_gate in [false, true] {
18167            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18168            runner.sync_to_base().await.unwrap();
18169            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18170            runner.state.reconcile_verification_head(&head);
18171            if waiting_on_gate {
18172                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18173                    head: head.clone(),
18174                    passed: true,
18175                    outcomes: Vec::new(),
18176                });
18177            }
18178            let cache = tmp.path().join("cache");
18179            runner.state.config.verify.e2e =
18180                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18181            runner.state.config.verify.gate = vec!["true".into()];
18182            runner.state.config.graph.timeout_resource_wait = 1;
18183            let owner =
18184                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18185            let guard =
18186                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18187                    crate::cache::AcquireOutcome::Acquired(g) => g,
18188                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18189                };
18190            runner.review_loop().await.unwrap();
18191            runner.gate().await.unwrap();
18192            assert!(!runner.state.gate_ran);
18193            assert_eq!(runner.state.gate_fixes.len(), 1);
18194            assert!(runner.state.resource_block.is_some());
18195            runner.merge().await.unwrap();
18196            assert!(runner.state.merge.is_none());
18197            drop(guard);
18198            runner.state = RunState::load(&runner.state.id).unwrap();
18199            // The driver clears the previous pass's resource stop on resume.
18200            runner.state.resource_block = None;
18201            runner.review_loop().await.unwrap();
18202            runner.gate().await.unwrap();
18203            assert!(runner.state.gate_allows_progress());
18204            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18205            assert_eq!(runner.state.gate_fixes.len(), 1);
18206        }
18207    }
18208
18209    #[cfg(unix)]
18210    #[tokio::test]
18211    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18212        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18213        runner.sync_to_base().await.unwrap();
18214        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18215        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18216        // Verification passes before pre_gate and fails on the mechanically changed head.
18217        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18218        runner.review_loop().await.unwrap();
18219        runner.gate().await.unwrap();
18220        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18221        assert_ne!(before, after);
18222        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18223        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18224        assert!(!runner.state.gate_allows_progress());
18225        runner.merge().await.unwrap();
18226        assert!(runner.state.merge.is_none());
18227        assert!(
18228            runner
18229                .state
18230                .verification_history
18231                .iter()
18232                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18233        );
18234
18235        // An incomplete review is still an independent hold on a changed head.
18236        runner.state.reviews[0].clean = false;
18237        runner.state.reviews[0].answered = 0;
18238        runner.state.config.verify.e2e = vec!["true".into()];
18239        runner.review_loop().await.unwrap();
18240        runner.gate().await.unwrap();
18241        runner.merge().await.unwrap();
18242        assert_eq!(runner.state.status, RunStatus::Blocked);
18243        assert!(runner.state.merge.is_none());
18244        assert!(
18245            runner
18246                .state
18247                .verification_blocking_cause()
18248                .unwrap()
18249                .contains("review blocked")
18250        );
18251    }
18252
18253    #[cfg(unix)]
18254    #[tokio::test]
18255    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18256        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18257        runner.state.gate_head = None;
18258        runner.state.config.verify.gate =
18259            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18260        runner.review_loop().await.unwrap();
18261        runner.gate().await.unwrap();
18262        assert_ne!(
18263            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18264            old
18265        );
18266        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18267        assert!(!runner.state.gate_allows_progress());
18268        assert_eq!(runner.state.status, RunStatus::Blocked);
18269        runner.merge().await.unwrap();
18270        assert!(runner.state.merge.is_none());
18271    }
18272
18273    #[cfg(unix)]
18274    #[tokio::test]
18275    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18276        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18277        let mut old_review = review_round(true, 0, 1, 1, false, true);
18278        old_review.head = head.clone();
18279        old_review.verified_head = Some(head.clone());
18280        runner.state.reviews = vec![old_review];
18281        runner.state.gate = vec![CommandOutcome {
18282            command: "old gate".into(),
18283            code: None,
18284            output_tail: "timed out after 2400s".into(),
18285            duration_ms: 2_400_000,
18286            resource_blocked: false,
18287        }];
18288        runner.state.gate_ran = true;
18289        runner.state.gate_head = Some(head.clone());
18290        runner.state.land_approval = Some(crate::run::LandApproval {
18291            question: "old-head-approval".into(),
18292            head: head.clone(),
18293        });
18294        runner.state.authorize_base_sync(&task).unwrap();
18295        runner.sync_to_base().await.unwrap();
18296        assert!(!runner.state.base_sync_blocked());
18297        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18298        assert_ne!(head, fresh);
18299        assert_eq!(runner.state.reviews.len(), 1);
18300        assert!(!runner.state.gate_ran);
18301        let saved = &runner.state.base_sync_recovery.authorizations[0];
18302        assert_eq!(saved.reviews[0].head, head);
18303        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18304        assert!(saved.gate_ran);
18305        assert!(!saved.gate[0].ok());
18306        runner.review_loop().await.unwrap();
18307        runner.gate().await.unwrap();
18308        assert_eq!(runner.state.reviews[0].head, head);
18309        assert_eq!(
18310            runner.state.reviews[0].verified_head.as_deref(),
18311            Some(head.as_str())
18312        );
18313        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18314        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18315        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18316        assert!(runner.state.gate_ran);
18317        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18318        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18319        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18320        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18321        assert_eq!(
18322            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18323            "retained change"
18324        );
18325    }
18326
18327    #[cfg(unix)]
18328    #[tokio::test]
18329    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18330        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18331        let queue = crate::queue::Queue::open();
18332        let mut owner = queue.get(&task).unwrap();
18333        owner.release_operator("test-resume", None);
18334        queue.put(&mut owner).unwrap();
18335        let generation = owner.resumption.as_ref().unwrap().generation;
18336        runner.state.task_instruction = Some(owner.instruction.clone());
18337        runner.state.driver_exited = true;
18338        runner.state.save().unwrap();
18339        let home = crate::run::home();
18340        let _claim = queue.claim(&task).unwrap();
18341        owner.resumption.as_mut().unwrap().at =
18342            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18343        queue.put(&mut owner).unwrap();
18344        assert!(
18345            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18346                .await
18347                .is_err(),
18348            "an approval from before the stop grants nothing"
18349        );
18350        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18351        queue.put(&mut owner).unwrap();
18352        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18353            .await
18354            .unwrap();
18355        assert!(
18356            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18357                .await
18358                .is_err()
18359        );
18360        runner.state = RunState::load(&runner.state.id).unwrap();
18361        assert_eq!(
18362            runner.state.base_sync_recovery.authorizations[0].generation,
18363            Some(generation)
18364        );
18365        runner.sync_to_base().await.unwrap();
18366        let repo = runner.state.repo.clone();
18367        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18368        git::git(&repo, &["checkout", "main"]).await.unwrap();
18369        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18370        git::git(&repo, &["add", "."]).await.unwrap();
18371        git::git(&repo, &["commit", "-m", "move base again"])
18372            .await
18373            .unwrap();
18374        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18375        git::git(&repo, &["checkout", "fix/retained"])
18376            .await
18377            .unwrap();
18378        runner.sync_to_base().await.unwrap();
18379        assert!(!runner.state.base_sync_blocked());
18380        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18381        assert_eq!(runner.state.base_episode_used(), 2);
18382        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18383        assert_ne!(fresh, head);
18384        assert!(!runner.state.base_recovery_pending());
18385        // Even replaying the generation with a later timestamp grants nothing.
18386        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18387        queue.put(&mut owner).unwrap();
18388        assert!(
18389            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18390                .await
18391                .is_err()
18392        );
18393        runner.state = RunState::load(&runner.state.id).unwrap();
18394        runner.sync_to_base().await.unwrap();
18395        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18396        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18397        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18398    }
18399
18400    #[cfg(unix)]
18401    #[tokio::test]
18402    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18403        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18404        runner.state.base_sync = None;
18405        runner.state.status = RunStatus::Blocked;
18406        let mut second = runner.state.candidates[0].clone();
18407        second.label = 'B';
18408        runner.state.candidates.push(second);
18409        assert!(runner.state.votes.is_empty());
18410        assert!(runner.state.deliberation.is_empty());
18411        runner.deliberate().await.unwrap();
18412        runner.vote().await.unwrap();
18413        assert_eq!(runner.state.status, RunStatus::Blocked);
18414        assert!(runner.state.votes.is_empty());
18415        assert!(runner.state.deliberation.is_empty());
18416    }
18417
18418    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18419    /// reads nothing else off it.
18420    fn runner_at(status: RunStatus) -> Runner {
18421        let mut state = RunState::new(
18422            PathBuf::from("/nonexistent/repo"),
18423            "main".to_owned(),
18424            "deadbeef".to_owned(),
18425            "task".to_owned(),
18426            Config::default(),
18427        );
18428        // These fixtures test execution flow, independent of host disk capacity.
18429        state.config.disk.min_free_bytes = 0;
18430        // These fixtures share one home and save concurrently. Short clock
18431        // IDs can collide, making an ownership test read another test's run.
18432        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18433        state.id = format!(
18434            "20000101-000000-{:04x}",
18435            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18436        );
18437        state.status = status;
18438        Runner {
18439            state,
18440            roles: ResolvedRoles {
18441                implementers: Vec::new(),
18442                judges: Vec::new(),
18443                reviewers: Vec::new(),
18444                fixer: None,
18445                conductor: conductor(),
18446                implementer_roster: Vec::new(),
18447                judge_roster: Vec::new(),
18448                reviewer_roster: Vec::new(),
18449            },
18450            sem: Arc::new(Semaphore::new(1)),
18451            pause: Pause::new(),
18452            interrupt: Pause::new(),
18453        }
18454    }
18455
18456    #[tokio::test]
18457    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18458        crate::run::pin_test_home();
18459        let temp = tempfile::tempdir().unwrap();
18460        let root = temp.path().canonicalize().unwrap();
18461        let repo = root.join("repo");
18462        std::fs::create_dir(&repo).unwrap();
18463        init_repo(&repo);
18464        git::git(&repo, &["config", "core.autocrlf", "false"])
18465            .await
18466            .unwrap();
18467        let mut runner = runner_at(RunStatus::Blocked);
18468        runner.state.repo = repo.clone();
18469        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18470        runner.state.config.agents = vec![conductor()];
18471        runner.state.config.graph.selected_implementer = Some("conductor".into());
18472        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18473        let (shell, command, setup_output) = if cfg!(windows) {
18474            (
18475                vec!["cmd".into(), "/C".into()],
18476                "echo ready>seat-ready",
18477                "ready\r\n",
18478            )
18479        } else {
18480            (
18481                vec!["sh".into(), "-c".into()],
18482                "printf ready > seat-ready",
18483                "ready",
18484            )
18485        };
18486        runner.state.config.verify.shell = Some(shell);
18487        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18488            run: Some(command.into()),
18489            ..Default::default()
18490        }];
18491        runner.state.save().unwrap();
18492        let path = runner.state.worktree_root().join("judge-1");
18493        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18494            .await
18495            .unwrap();
18496        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18497        git::git(&path, &["add", "."]).await.unwrap();
18498        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18499        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18500            .await
18501            .unwrap();
18502        git::git(&path, &["checkout", "--detach", &head])
18503            .await
18504            .unwrap();
18505        let evidence = crate::fold_evidence::snapshot(
18506            &runner.state,
18507            &crate::run::home().canonicalize().unwrap(),
18508            &head,
18509            64 * 1024 * 1024,
18510        )
18511        .await
18512        .unwrap();
18513        let mut contents = BTreeMap::new();
18514        contents.insert(
18515            repo.join("logs/HEAD"),
18516            format!("{head} {head} fixture\n").into_bytes(),
18517        );
18518        let record = serde_json::json!({
18519            "version":1, "run":runner.state.id, "source":path, "head":head,
18520            "evidence":evidence, "admin":{}, "admin_contents":contents,
18521            "history":{head.clone():evidence}, "phase":"evicted"
18522        });
18523        std::fs::write(
18524            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18525            serde_json::to_vec(&record).unwrap(),
18526        )
18527        .unwrap();
18528        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18529        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18530            .await
18531            .unwrap();
18532        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18533        assert!(
18534            !git::git_raw(&repo, &["cat-file", "-e", &head])
18535                .await
18536                .unwrap()
18537                .ok()
18538        );
18539        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18540        let error = resumed.execute().await.unwrap_err();
18541        assert!(
18542            error
18543                .to_string()
18544                .contains("selected implementation is held"),
18545            "{error:#}"
18546        );
18547        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18548        assert_eq!(
18549            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18550            setup_output
18551        );
18552        assert_eq!(
18553            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18554            "unrelated seat"
18555        );
18556        let error = resumed.execute().await.unwrap_err();
18557        assert!(
18558            error
18559                .to_string()
18560                .contains("selected implementation is held"),
18561            "{error:#}"
18562        );
18563        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18564        git::git(&path, &["add", "."]).await.unwrap();
18565        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18566        let next = git::git(
18567            &path,
18568            &[
18569                "commit-tree",
18570                &next_tree,
18571                "-p",
18572                &head,
18573                "-m",
18574                "next review checkpoint",
18575            ],
18576        )
18577        .await
18578        .unwrap();
18579        git::git(&path, &["checkout", "--detach", &next])
18580            .await
18581            .unwrap();
18582        resumed.state.reviews.push(
18583            serde_json::from_value(serde_json::json!({
18584                "round":1, "head":next, "reviews":[]
18585            }))
18586            .unwrap(),
18587        );
18588        resumed.state.save().unwrap();
18589        let error = resumed.execute().await.unwrap_err();
18590        assert!(
18591            error
18592                .to_string()
18593                .contains("selected implementation is held"),
18594            "{error:#}"
18595        );
18596        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18597    }
18598
18599    #[tokio::test]
18600    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18601        crate::run::pin_test_home();
18602        let mut runner = runner_at(RunStatus::Gating);
18603        runner.state.save().unwrap();
18604        let id = runner.state.id.clone();
18605        let passes = runner.state.passes.len();
18606        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18607        let mut claiming = Box::pin(runner.claim_driver());
18608        std::future::poll_fn(|cx| {
18609            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18610            std::task::Poll::Ready(())
18611        })
18612        .await;
18613        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18614        drop(owner);
18615        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18616            .await
18617            .unwrap()
18618            .unwrap();
18619        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18620        assert!(execution.runner.state.resource_block.is_none());
18621        assert_eq!(
18622            execution.runner.state.liveness(false),
18623            crate::run::Liveness::Live
18624        );
18625        drop(execution);
18626        assert!(RunState::load(&id).unwrap().driver_exited);
18627    }
18628
18629    #[tokio::test]
18630    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18631        crate::run::pin_test_home();
18632        let mut runner = runner_at(RunStatus::Gating);
18633        runner.state.save().unwrap();
18634        // run.json can commit, but the subsequent lease metadata write fails.
18635        // Cleanup must run after releasing the short ownership lock.
18636        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18637        assert!(runner.claim_driver().await.is_err());
18638        assert!(runner.state.driver_exited);
18639        assert_eq!(runner.state.driver_pid, None);
18640        let disk = RunState::load(&runner.state.id).unwrap();
18641        assert!(disk.driver_exited);
18642        assert_eq!(disk.driver_pid, None);
18643        assert_eq!(disk.status, RunStatus::Gating);
18644    }
18645
18646    #[tokio::test]
18647    async fn an_early_graph_error_releases_its_driver() {
18648        crate::run::pin_test_home();
18649        let mut runner = runner_at(RunStatus::Blocked);
18650        runner.state.config.graph.selected_implementer = Some("selected".into());
18651        runner.state.save().unwrap();
18652        assert!(runner.execute().await.is_err());
18653        let disk = RunState::load(&runner.state.id).unwrap();
18654        assert!(disk.driver_exited);
18655        assert_eq!(disk.driver_pid, None);
18656        assert_eq!(disk.status, RunStatus::Blocked);
18657    }
18658
18659    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18660        crate::run::MergeRepair {
18661            question: "repair-question".into(),
18662            head: "old-head".into(),
18663            pr: "https://github.com/o/r/pull/42".into(),
18664            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18665            gate: vec![outcome(Some(0), false)],
18666            gate_head: Some("old-head".into()),
18667            gate_fixes: Vec::new(),
18668            pre_gate: Vec::new(),
18669            phase,
18670            failure: None,
18671        }
18672    }
18673
18674    #[tokio::test]
18675    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18676        crate::run::pin_test_home();
18677        let mut runner = runner_at(RunStatus::Landing);
18678        runner.state.merge_repairs.push(merge_repair_request(
18679            crate::run::MergeRepairPhase::Requested,
18680        ));
18681        runner.state.save().unwrap();
18682        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18683        assert!(
18684            runner
18685                .execute()
18686                .await
18687                .unwrap_err()
18688                .to_string()
18689                .contains("already running")
18690        );
18691        assert_eq!(
18692            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18693            crate::run::MergeRepairPhase::Requested
18694        );
18695        drop(claim);
18696        let mut disk = runner.state.clone();
18697        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18698        disk.save().unwrap();
18699        assert!(
18700            runner
18701                .execute()
18702                .await
18703                .unwrap_err()
18704                .to_string()
18705                .contains("decision changed")
18706        );
18707        assert_eq!(
18708            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18709            crate::run::MergeRepairPhase::Complete
18710        );
18711    }
18712
18713    #[test]
18714    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18715        crate::run::pin_test_home();
18716        let mut runner = runner_at(RunStatus::Landing);
18717        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18718        runner.state.reviews = request.reviews.clone();
18719        runner.state.gate = request.gate.clone();
18720        runner.state.gate_ran = true;
18721        runner.state.land_approval = Some(crate::run::LandApproval {
18722            question: "old".into(),
18723            head: "old-head".into(),
18724        });
18725        runner.state.merge_repairs.push(request);
18726        runner.begin_merge_repair_review(0).unwrap();
18727        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18728        assert!(!runner.state.gate_status().ok());
18729        assert!(runner.state.land_approval.is_none());
18730        let disk = RunState::load(&runner.state.id).unwrap();
18731        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18732        assert!(disk.merge_repairs[0].gate[0].ok());
18733        assert_eq!(
18734            disk.merge_repairs[0].phase,
18735            crate::run::MergeRepairPhase::Reviewing
18736        );
18737        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18738        assert!(disk.reviews.is_empty());
18739    }
18740
18741    #[tokio::test]
18742    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18743        crate::run::pin_test_home();
18744        for phase in [
18745            crate::run::MergeRepairPhase::Fixing,
18746            crate::run::MergeRepairPhase::Requested,
18747        ] {
18748            let mut runner = runner_at(RunStatus::Landing);
18749            runner.state.config.graph.land_rounds = 0;
18750            runner.state.merge_repairs.push(merge_repair_request(phase));
18751            runner.state.save().unwrap();
18752            assert!(!runner.repair_merge_review().await.unwrap());
18753            assert_eq!(runner.state.status, RunStatus::Blocked);
18754            assert_eq!(
18755                runner.state.merge_repairs[0].phase,
18756                crate::run::MergeRepairPhase::Failed
18757            );
18758            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18759            assert!(
18760                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18761                    "interrupted"
18762                } else {
18763                    "budget exhausted"
18764                })
18765            );
18766            assert!(runner.state.merge.is_none());
18767        }
18768    }
18769
18770    /// `park_here` folding in the reason `Pause::park_because` recorded -
18771    /// this is what lets an operator reading a run's events tell an
18772    /// interrupt-driven park from an ordinary shutdown park.
18773    #[test]
18774    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18775        crate::run::pin_test_home();
18776        let mut runner = runner_at(RunStatus::Implementing);
18777        let interrupt = Pause::new();
18778        runner.watch_interrupt(interrupt.clone());
18779
18780        interrupt.park_because("task a1b2 asked to run first");
18781
18782        assert!(runner.park_here().expect("park_here"));
18783        assert!(runner.state.parked);
18784        let last = runner.state.events.last().expect("a park event");
18785        assert_eq!(last.node, "park");
18786        assert!(
18787            last.message.contains("task a1b2 asked to run first"),
18788            "expected the interrupt reason in {:?}",
18789            last.message
18790        );
18791    }
18792
18793    /// The cause is stored on the pass when it parks, and the first cause a
18794    /// shared pause was given is the one that stays.
18795    #[test]
18796    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18797        crate::run::pin_test_home();
18798        let mut runner = runner_at(RunStatus::Implementing);
18799        runner.state.begin_pass();
18800        let shutdown = Pause::new();
18801        runner.on_pause(shutdown.clone());
18802        let upgrade = ParkCause::Upgrade {
18803            from: "0.121.0".to_owned(),
18804            to: Some("0.121.1".to_owned()),
18805        };
18806        shutdown.park_for(upgrade.clone());
18807        shutdown.park_for(ParkCause::Operator);
18808
18809        assert!(runner.park_here().expect("park_here"));
18810        assert_eq!(
18811            runner.state.passes[0].end,
18812            Some(crate::run::PassEnd::Parked { cause: upgrade })
18813        );
18814        let last = runner.state.events.last().expect("a park event");
18815        assert!(last.message.contains("0.121.0"), "{}", last.message);
18816    }
18817
18818    #[tokio::test]
18819    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18820        crate::run::pin_test_home();
18821        let mut runner = runner_at(RunStatus::Reviewing);
18822        runner.state.save().unwrap();
18823        let mut execution = runner.claim_driver().await.expect("first execution");
18824        assert_eq!(execution.runner.state.passes.len(), 1);
18825        execution
18826            .driver
18827            .finish(&mut execution.runner.state)
18828            .await
18829            .unwrap();
18830    }
18831
18832    #[tokio::test]
18833    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18834        crate::run::pin_test_home();
18835        let mut runner = runner_at(RunStatus::Reviewing);
18836        runner.state.config.agents = vec![conductor()];
18837        runner.state.config.graph.timeout_resource_wait = 0;
18838        runner.state.save().unwrap();
18839        let id = runner.state.id.clone();
18840        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18841        assert!(runner.claim_driver().await.is_err());
18842        drop(owner);
18843        drop(runner);
18844        let mut resumed = Runner::resume(&id).unwrap();
18845        assert_eq!(resumed.state.driver_claimed, Some(false));
18846        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18847        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18848        let mut execution = resumed
18849            .claim_driver()
18850            .await
18851            .expect("retry initial claim after reload");
18852        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18853        assert_eq!(execution.runner.state.passes.len(), 1);
18854        execution
18855            .driver
18856            .finish(&mut execution.runner.state)
18857            .await
18858            .unwrap();
18859    }
18860
18861    #[tokio::test]
18862    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18863        crate::run::pin_test_home();
18864        let mut runner = runner_at(RunStatus::Reviewing);
18865        runner.state.config.agents = vec![conductor()];
18866        runner.state.save().unwrap();
18867        let id = runner.state.id.clone();
18868        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18869        let mut resumed = Runner::resume(&id).unwrap();
18870        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18871        assert!(resumed.claim_driver().await.is_err());
18872        assert!(RunState::load(&id).unwrap().passes.is_empty());
18873        drop(lease);
18874        let mut execution = resumed.claim_driver().await.unwrap();
18875        execution
18876            .driver
18877            .finish(&mut execution.runner.state)
18878            .await
18879            .unwrap();
18880    }
18881
18882    #[tokio::test]
18883    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18884        crate::run::pin_test_home();
18885        let mut runner = runner_at(RunStatus::Reviewing);
18886        let state = &mut runner.state;
18887        state.config.agents = vec![conductor()];
18888        state.driver_claimed = None;
18889        state.driver_pid = Some(std::process::id());
18890        state.driver_started_at = None;
18891        state.begin_pass();
18892        state.save().unwrap();
18893        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18894        std::fs::write(
18895            state.dir().join("driver.json"),
18896            serde_json::to_vec(&serde_json::json!({
18897                "pid": state.driver_pid, "started_at": null,
18898                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18899            }))
18900            .unwrap(),
18901        )
18902        .unwrap();
18903        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18904        drop(lease);
18905        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18906        let mut resumed = Runner::resume(&state.id).unwrap();
18907        assert!(!resumed.state.driver_exited);
18908        let mut execution = resumed
18909            .claim_driver()
18910            .await
18911            .expect("lease proves previous execution ended");
18912        assert_eq!(execution.runner.state.passes.len(), 2);
18913        execution
18914            .driver
18915            .finish(&mut execution.runner.state)
18916            .await
18917            .unwrap();
18918    }
18919
18920    #[tokio::test]
18921    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18922        crate::run::pin_test_home();
18923        let mut runner = runner_at(RunStatus::Reviewing);
18924        let mut json = serde_json::to_value(&runner.state).unwrap();
18925        json.as_object_mut().unwrap().remove("driver_claimed");
18926        runner.state = serde_json::from_value(json).unwrap();
18927        assert_eq!(runner.state.driver_claimed, None);
18928        runner.state.save().unwrap();
18929        let mut execution = runner
18930            .claim_driver()
18931            .await
18932            .expect("the execution lease, not a missing marker, excludes a second driver");
18933        assert_eq!(execution.runner.state.passes.len(), 1);
18934        execution
18935            .driver
18936            .finish(&mut execution.runner.state)
18937            .await
18938            .unwrap();
18939    }
18940
18941    #[tokio::test]
18942    async fn driver_admission_refuses_a_proven_live_driver() {
18943        crate::run::pin_test_home();
18944        let mut runner = runner_at(RunStatus::Reviewing);
18945        let pid = std::process::id();
18946        runner.state.driver_pid = Some(pid);
18947        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18948        assert!(runner.state.driver_started_at.is_some());
18949        runner.state.save().unwrap();
18950        let error = runner
18951            .claim_driver()
18952            .await
18953            .err()
18954            .expect("a live driver must refuse");
18955        assert!(error.to_string().contains("live driver"), "{error:#}");
18956        let saved = RunState::load(&runner.state.id).unwrap();
18957        assert_eq!(saved.driver_pid, Some(pid));
18958        assert_eq!(saved.status, RunStatus::Reviewing);
18959        assert!(saved.passes.is_empty());
18960    }
18961
18962    #[test]
18963    fn parking_resource_halt_persists_the_pass_once() {
18964        crate::run::pin_test_home();
18965        let mut runner = runner_at(RunStatus::Reviewing);
18966        runner.state.begin_pass();
18967        runner.state.resource_block = Some(crate::run::ResourceBlock {
18968            node: "fix".into(),
18969            kind: "shared build cache".into(),
18970            detail: "cancelled".into(),
18971            at: Timestamp::now(),
18972            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18973        });
18974        let cause = ParkCause::Upgrade {
18975            from: "old".into(),
18976            to: Some("new".into()),
18977        };
18978        runner.pause.park_for(cause.clone());
18979        // Both direct fixer exits and graph resource boundaries must complete
18980        // parking, rather than returning a stale Reviewing record.
18981        runner.stop_for_resource().unwrap();
18982        assert!(runner.resource_halt().unwrap());
18983        assert!(runner.park_here().unwrap());
18984        let saved = RunState::load(&runner.state.id).unwrap();
18985        assert!(saved.parked);
18986        assert_eq!(saved.status, RunStatus::Reviewing);
18987        assert!(saved.passes[0].ended_at.is_some());
18988        assert_eq!(
18989            saved.passes[0].end,
18990            Some(crate::run::PassEnd::Parked {
18991                cause: cause.clone()
18992            })
18993        );
18994        assert_eq!(
18995            saved.resource_block.unwrap().cause,
18996            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18997        );
18998        assert_eq!(
18999            saved
19000                .events
19001                .iter()
19002                .filter(|event| event.node == "park")
19003                .count(),
19004            1
19005        );
19006    }
19007
19008    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
19009    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
19010    /// cleared) must not make a *different* run - one only watching its own,
19011    /// unshared interrupt `Pause` - see itself as parked. If a future change
19012    /// ever collapsed these back into one handle, the interrupt scheduler
19013    /// would park every run for the rest of the daemon's life, not just the
19014    /// one it meant to interrupt.
19015    #[test]
19016    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19017        crate::run::pin_test_home();
19018        let mut runner = runner_at(RunStatus::Implementing);
19019        let shutdown = Pause::new();
19020        runner.on_pause(shutdown.clone());
19021        let interrupt = Pause::new();
19022        runner.watch_interrupt(interrupt.clone());
19023
19024        // Nobody has asked for anything yet.
19025        assert!(!runner.park_here().expect("park_here"));
19026        assert!(!runner.state.parked);
19027
19028        // Only the interrupt handle fires; the shutdown handle stays clear.
19029        interrupt.park_because("test");
19030        assert!(!shutdown.parked());
19031        assert!(runner.park_here().expect("park_here"));
19032    }
19033
19034    /// The property every prior attempt at this feature failed to pin down:
19035    /// asking a run to park while one of its nodes has a real, in-flight
19036    /// async operation running (an agent call, in production) must not cut
19037    /// that operation short. `park_here` is only ever consulted *between*
19038    /// `execute`'s node calls - see its own doc - so nothing inside a node
19039    /// can observe a park request until the node itself returns. This proves
19040    /// that structurally, with real `tokio` concurrency and a channel
19041    /// handshake (never a sleep, which would only prove "usually", not
19042    /// "cannot"): the "node" below reports that it has genuinely started,
19043    /// and only then is the park requested; the node still has to be told to
19044    /// finish before `park_here` is ever called, exactly mirroring every
19045    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
19046    /// in `execute`.
19047    #[tokio::test]
19048    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19049        crate::run::pin_test_home();
19050        let mut runner = runner_at(RunStatus::Implementing);
19051        let interrupt = Pause::new();
19052        runner.watch_interrupt(interrupt.clone());
19053
19054        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19055        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19056
19057        // Stands in for one node's in-flight agent call: it proves it has
19058        // genuinely started, then blocks - exactly as a spawned CLI process
19059        // does - until told to finish.
19060        let node = async move {
19061            started_tx.send(()).expect("send started");
19062            finish_rx.await.expect("recv finish");
19063            "node finished"
19064        };
19065
19066        let interrupter = async move {
19067            started_rx.await.expect("recv started");
19068            // The call is now genuinely in flight. Ask it to park.
19069            interrupt.park_because("higher-priority task waiting");
19070            // Nothing the node does can observe this yet - there is no
19071            // check inside it, by construction - so let the executor run
19072            // anything pending and then let the node finish on its own.
19073            tokio::task::yield_now().await;
19074            finish_tx.send(()).expect("send finish");
19075        };
19076
19077        let (node_result, ()) = tokio::join!(node, interrupter);
19078        assert_eq!(
19079            node_result, "node finished",
19080            "the in-flight call ran to completion"
19081        );
19082
19083        // Only now, at the boundary the real `execute` would check right
19084        // after this node, does the park take effect.
19085        assert!(runner.park_here().expect("park_here"));
19086        assert!(runner.state.parked);
19087    }
19088
19089    /// A run parked mid-competition carries every field it had accumulated
19090    /// through the exact same disk round-trip an ordinary resume uses -
19091    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
19092    /// Nothing about parking for an interrupt is a special case of that path;
19093    /// this is what proves it rather than assuming it.
19094    #[test]
19095    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19096        crate::run::pin_test_home();
19097        let mut runner = runner_at(RunStatus::Judging);
19098        // `Runner::resume` re-resolves roles from the saved config, which
19099        // refuses an empty roster - give it the same minimal one `conductor`
19100        // itself uses.
19101        runner.state.config.agents = vec![conductor()];
19102        runner.state.candidates = vec![Candidate {
19103            index: 0,
19104            label: 'A',
19105            agent: "alpha".to_owned(),
19106            branch: "magi/x/A".to_owned(),
19107            worktree: PathBuf::from("/nonexistent/worktree"),
19108            summary: "did the thing".to_owned(),
19109            stat: "1 file changed".to_owned(),
19110            files: 1,
19111            commits: 1,
19112            empty: false,
19113            failed: None,
19114            verified_noop: None,
19115            duration_ms: 1234,
19116            folded: false,
19117        }];
19118        let run_id = runner.state.id.clone();
19119
19120        let interrupt = Pause::new();
19121        runner.watch_interrupt(interrupt.clone());
19122        interrupt.park_because("task c3d4 asked to run first");
19123        assert!(runner.park_here().expect("park_here"));
19124
19125        let resumed = Runner::resume(&run_id).expect("resume");
19126        assert_eq!(resumed.state.candidates.len(), 1);
19127        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19128        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19129        assert_eq!(resumed.state.status, runner.state.status);
19130        assert!(
19131            resumed.state.parked,
19132            "still parked until `execute` actually walks the graph again"
19133        );
19134        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19135    }
19136
19137    /// A fresh open question on `run`, stored and handed back for assertions.
19138    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19139        let mut q = ask::Question::new(
19140            run.to_owned(),
19141            "implement".to_owned(),
19142            "impl-A".to_owned(),
19143            "Which storage backend should the cache use?".to_owned(),
19144            String::new(),
19145            vec!["SQLite".to_owned(), "Redis".to_owned()],
19146        );
19147        store.put(&mut q).unwrap();
19148        q
19149    }
19150
19151    #[test]
19152    fn a_failed_runs_open_question_is_abandoned() {
19153        ask_test_home();
19154        let store = ask::Questions::open();
19155        let mut runner = runner_at(RunStatus::Failed);
19156        let run = runner.state.id.clone();
19157        let q = ask_open_question(&store, &run);
19158
19159        runner.settle_questions();
19160
19161        let back = store.get(&q.id).unwrap();
19162        assert!(
19163            !back.status.open(),
19164            "the seat that asked died with the run; nobody is left to read an answer"
19165        );
19166        assert!(
19167            back.detail.contains(&run) && back.detail.contains("failed"),
19168            "the reason names what the run became, not just that it is gone: {}",
19169            back.detail
19170        );
19171    }
19172
19173    #[test]
19174    fn a_merged_runs_open_question_is_abandoned_too() {
19175        ask_test_home();
19176        let store = ask::Questions::open();
19177        // A run that finishes cleanly still leaves nobody to read an answer -
19178        // this is not only a failure-path cleanup.
19179        for status in [RunStatus::Merged, RunStatus::Ready] {
19180            let mut runner = runner_at(status);
19181            let run = runner.state.id.clone();
19182            let q = ask_open_question(&store, &run);
19183
19184            runner.settle_questions();
19185
19186            let back = store.get(&q.id).unwrap();
19187            assert!(
19188                !back.status.open(),
19189                "{status:?} run's question must not outlive the run"
19190            );
19191        }
19192    }
19193
19194    #[test]
19195    fn a_still_resumable_runs_open_question_is_left_alone() {
19196        ask_test_home();
19197        let store = ask::Questions::open();
19198        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19199        // review round and the seat sessions are all still on disk — so a
19200        // question asked mid-round may yet get a real answer from a real
19201        // resume. Sweeping it here would be exactly the failure mode this
19202        // whole feature exists to avoid on the other side.
19203        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19204            let mut runner = runner_at(status);
19205            let run = runner.state.id.clone();
19206            let q = ask_open_question(&store, &run);
19207
19208            runner.settle_questions();
19209
19210            let back = store.get(&q.id).unwrap();
19211            assert!(
19212                back.status.open(),
19213                "{status:?} is still alive; the question must still be waiting"
19214            );
19215        }
19216    }
19217
19218    #[test]
19219    fn settle_questions_never_touches_an_already_answered_question() {
19220        ask_test_home();
19221        let store = ask::Questions::open();
19222        let mut runner = runner_at(RunStatus::Failed);
19223        let run = runner.state.id.clone();
19224        let mut q = ask_open_question(&store, &run);
19225        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19226            .unwrap();
19227        store.put(&mut q).unwrap();
19228
19229        // Called twice, the way a crash-recovered daemon reclaim and the
19230        // graph's own cleanup both can for the same run — `abandon_for_run`
19231        // only ever touches what is still open, so this must be inert both
19232        // times, not merely the second.
19233        runner.settle_questions();
19234        runner.settle_questions();
19235
19236        let back = store.get(&q.id).unwrap();
19237        assert_eq!(
19238            back.status,
19239            ask::QuestionStatus::Answered,
19240            "a real answer is a decision on record, never overwritten by a sweep"
19241        );
19242    }
19243
19244    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19245    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19246    /// without merging, whose winner is still the operator's answer to read.
19247    /// Nothing previously called `fold_run` itself with a real `tally`, so
19248    /// this is the first test to pin down the one distinction the whole
19249    /// automatic-fold feature depends on: the winner's worktree and branch
19250    /// must survive, everything else sharing the run's worktree bay - a
19251    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19252    /// second sweep treats every non-winner directory under the bay alike -
19253    /// must not.
19254    #[tokio::test]
19255    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19256        crate::run::pin_test_home();
19257        let tmp = tempfile::tempdir().expect("tempdir");
19258        let repo = tmp.path().join("repo");
19259        std::fs::create_dir_all(&repo).unwrap();
19260        init_repo(&repo);
19261
19262        let mut config = Config::default();
19263        config.graph.worktree_root = Some(tmp.path().join("wt"));
19264
19265        let mut state = RunState::new(
19266            repo.clone(),
19267            "main".to_owned(),
19268            "deadbeef".to_owned(),
19269            "task".to_owned(),
19270            config,
19271        );
19272        let root = state.worktree_root();
19273        let wt_a = root.join("cand-A");
19274        let wt_b = root.join("cand-B");
19275        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19276            .await
19277            .expect("worktree A");
19278        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19279            .await
19280            .expect("worktree B");
19281
19282        state.candidates = vec![
19283            Candidate {
19284                index: 0,
19285                label: 'A',
19286                agent: "alpha".to_owned(),
19287                branch: "magi/x/A".to_owned(),
19288                worktree: wt_a.clone(),
19289                summary: String::new(),
19290                stat: String::new(),
19291                files: 0,
19292                commits: 0,
19293                empty: false,
19294                failed: None,
19295                verified_noop: None,
19296                duration_ms: 0,
19297                folded: false,
19298            },
19299            Candidate {
19300                index: 1,
19301                label: 'B',
19302                agent: "beta".to_owned(),
19303                branch: "magi/x/B".to_owned(),
19304                worktree: wt_b.clone(),
19305                summary: String::new(),
19306                stat: String::new(),
19307                files: 0,
19308                commits: 0,
19309                empty: false,
19310                failed: None,
19311                verified_noop: None,
19312                duration_ms: 0,
19313                folded: false,
19314            },
19315        ];
19316        state.tally = Some(Tally {
19317            first_choice: BTreeMap::from([('A', 1)]),
19318            borda: BTreeMap::new(),
19319            winner: 'A',
19320            rankings: 1,
19321            unanimous_initial: true,
19322            deliberated: false,
19323            changed_votes: 0,
19324            unanimous_final: true,
19325            tie_break: None,
19326            judges: 1,
19327            present: 1,
19328            quorum: 1,
19329            met_quorum: true,
19330            uncontested: None,
19331        });
19332        state.status = RunStatus::Ready;
19333
19334        fold_run(&mut state, false, &crate::run::home())
19335            .await
19336            .expect("fold_run");
19337
19338        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19339        assert!(
19340            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19341            "the unmerged winner's branch survives"
19342        );
19343        assert!(
19344            !state.candidates[0].folded,
19345            "the winner is not marked folded"
19346        );
19347
19348        assert!(!wt_b.exists(), "the loser's worktree is removed");
19349        assert!(
19350            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19351            "the loser's branch is removed"
19352        );
19353        assert!(state.candidates[1].folded, "the loser is marked folded");
19354    }
19355
19356    #[tokio::test]
19357    #[cfg(unix)]
19358    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19359        use std::os::fd::AsRawFd as _;
19360        use std::time::Duration;
19361
19362        let tmp = tempfile::tempdir().unwrap();
19363        let repo = tmp.path().join("repo");
19364        std::fs::create_dir_all(&repo).unwrap();
19365        init_repo(&repo);
19366        let common = repo.join(".git");
19367        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19368        let home = tmp.path().join("home");
19369        let mut config = Config::default();
19370        config.coordination = crate::config::test_coordination();
19371        config.graph.worktree_root = Some(tmp.path().join("wt"));
19372        config.graph.timeout_resource_wait = 1;
19373        let mut state = RunState::new(
19374            repo.clone(),
19375            "main".into(),
19376            "deadbeef".into(),
19377            "task".into(),
19378            config,
19379        );
19380        let root = state.worktree_root();
19381        let candidate = root.join("cand-A");
19382        let reviewer = root.join("review-1");
19383        let branch = state.branch_for('A');
19384        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19385            .await
19386            .unwrap();
19387        git::worktree_add_detached(&repo, &reviewer, "main")
19388            .await
19389            .unwrap();
19390        git::acquire_worktree_config(&repo).await.unwrap();
19391        state.enabled_worktree_config = true;
19392        state.status = RunStatus::Ready;
19393        state.candidates.push(Candidate {
19394            index: 0,
19395            label: 'A',
19396            agent: "implement".into(),
19397            branch: branch.clone(),
19398            worktree: candidate.clone(),
19399            summary: String::new(),
19400            stat: String::new(),
19401            files: 0,
19402            commits: 0,
19403            empty: false,
19404            failed: None,
19405            verified_noop: None,
19406            duration_ms: 0,
19407            folded: false,
19408        });
19409        state.save_under(&home).unwrap();
19410        let config_before = std::fs::read(common.join("config")).unwrap();
19411        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19412            .await
19413            .unwrap();
19414        // A separate OS handle bypasses the launcher's local gate, exactly as
19415        // an external updater does, and must still prevent all folding work.
19416        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19417        assert_eq!(
19418            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19419            0
19420        );
19421        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19422        assert!(
19423            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19424                .await
19425                .is_err()
19426        );
19427        assert!(candidate.exists());
19428        assert!(reviewer.exists());
19429        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19430        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19431        assert_eq!(
19432            git::git(&repo, &["worktree", "list", "--porcelain"])
19433                .await
19434                .unwrap(),
19435            worktrees_before
19436        );
19437        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19438            .await
19439            .unwrap()
19440            .unwrap_err();
19441        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19442        drop(fold);
19443        assert!(candidate.exists() && reviewer.exists());
19444        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19445        assert_eq!(
19446            state.resource_block.as_ref().unwrap().kind,
19447            "repository coordination"
19448        );
19449        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19450        drop(external);
19451        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19452        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19453            .await
19454            .unwrap()
19455            .unwrap();
19456        drop(fold);
19457        assert!(removed.contains(&branch));
19458        assert!(!root.exists());
19459        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19460        assert!(state.candidates[0].folded);
19461        assert!(!state.enabled_worktree_config);
19462        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19463        let enabled = git::git_raw(
19464            &repo,
19465            &["config", "--local", "--get", "extensions.worktreeConfig"],
19466        )
19467        .await
19468        .unwrap();
19469        assert!(!enabled.ok(), "the last config reference was released");
19470    }
19471
19472    /// A branch handed to a later run is that run's (and its pull request's):
19473    /// folding the run that released it must not delete it.
19474    #[tokio::test]
19475    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19476        let tmp = tempfile::tempdir().expect("tempdir");
19477        let repo = tmp.path().join("repo");
19478        std::fs::create_dir_all(&repo).unwrap();
19479        init_repo(&repo);
19480        let home = tmp.path().join("home");
19481
19482        let mut config = Config::default();
19483        config.graph.worktree_root = Some(tmp.path().join("wt"));
19484        let mut state = RunState::new(
19485            repo.clone(),
19486            "main".to_owned(),
19487            "deadbeef".to_owned(),
19488            "task".to_owned(),
19489            config,
19490        );
19491        // The worktree is already gone (released); the branch survives.
19492        git::git(&repo, &["branch", "magi/x/A", "main"])
19493            .await
19494            .expect("branch");
19495        state.candidates = vec![Candidate {
19496            index: 0,
19497            label: 'A',
19498            agent: "alpha".to_owned(),
19499            branch: "magi/x/A".to_owned(),
19500            worktree: state.worktree_root().join("cand-A"),
19501            summary: String::new(),
19502            stat: String::new(),
19503            files: 0,
19504            commits: 0,
19505            empty: false,
19506            failed: None,
19507            verified_noop: None,
19508            duration_ms: 0,
19509            folded: true,
19510        }];
19511        state.released_to = Some("20260901-000000-new1".to_owned());
19512        state.released_branches = vec!["magi/x/A".to_owned()];
19513
19514        fold_run(&mut state, true, &home).await.expect("fold_run");
19515        state.status = RunStatus::Merged;
19516        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19517        state.merge = Some(crate::run::MergeOutcome {
19518            mode: MergeMode::Pr,
19519            ok: true,
19520            detail: "confirmed merged".into(),
19521            empty: false,
19522        });
19523        fold_run(&mut state, true, &home)
19524            .await
19525            .expect("merged fold");
19526
19527        assert!(
19528            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19529            "the handed-over branch survives a fold"
19530        );
19531    }
19532
19533    /// A winner with nothing ahead of the base is caught before `gh` is ever
19534    /// asked for a pull request, and the message carries what the task's
19535    /// references resolved to.
19536    #[tokio::test]
19537    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19538        let tmp = tempfile::tempdir().expect("tempdir");
19539        let repo = tmp.path().join("repo");
19540        std::fs::create_dir_all(&repo).unwrap();
19541        init_repo(&repo);
19542        let run = |args: &[&str]| {
19543            let out = std::process::Command::new("git")
19544                .quiet()
19545                .args(args)
19546                .current_dir(&repo)
19547                .output()
19548                .expect("spawn git");
19549            assert!(out.status.success(), "git {args:?}");
19550        };
19551        run(&["branch", "magi/x/A"]);
19552        run(&["checkout", "-q", "-b", "magi/x/B"]);
19553        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19554        run(&["add", "-A"]);
19555        run(&["commit", "-q", "-m", "work"]);
19556        run(&["checkout", "-q", "main"]);
19557
19558        // A pull request is compared against the remote's base, so the
19559        // fixture needs one. Before it exists nothing can be read, and the
19560        // answer must be "not empty".
19561        let probe = RunState::new(
19562            repo.clone(),
19563            "main".to_owned(),
19564            "deadbeef".to_owned(),
19565            "task".to_owned(),
19566            Config::default(),
19567        );
19568        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19569        let bare = tmp.path().join("origin.git");
19570        let out = std::process::Command::new("git")
19571            .quiet()
19572            .args(["init", "-q", "--bare"])
19573            .arg(&bare)
19574            .output()
19575            .expect("spawn git");
19576        assert!(out.status.success(), "git init --bare");
19577        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19578        run(&["push", "-q", "origin", "main"]);
19579
19580        let mut state = RunState::new(
19581            repo.clone(),
19582            "main".to_owned(),
19583            "deadbeef".to_owned(),
19584            "task".to_owned(),
19585            Config::default(),
19586        );
19587        state.seeds = vec![refs::Seed {
19588            token: "magi/27b2/A".to_owned(),
19589            kind: refs::SeedKind::Unresolved,
19590            sha: String::new(),
19591            branch: true,
19592            detail: "no branch or commit named magi/27b2/A".to_owned(),
19593        }];
19594
19595        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19596        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19597        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19598        let detail = empty_candidate_detail(&state, "main");
19599        assert!(detail.starts_with("empty candidate"), "{detail}");
19600        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19601        assert!(detail.contains("--start-from"), "{detail}");
19602    }
19603
19604    /// `status == Ready` used to be read as "this is the harmless
19605    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19606    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19607    /// run's PR was closed without merging — and reentering `merge` with
19608    /// `mode` still `Pr` does not know the difference, so it pushed and
19609    /// opened a second pull request. `mode == Local` reproduces the same
19610    /// blind spot without a network call: reentry must not attempt another
19611    /// git merge once this node has already recorded an outcome.
19612    #[tokio::test]
19613    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19614        let tmp = tempfile::tempdir().expect("tempdir");
19615        let repo = tmp.path().join("repo");
19616        std::fs::create_dir_all(&repo).unwrap();
19617        init_repo(&repo);
19618
19619        let mut config = Config::default();
19620        config.merge.mode = MergeMode::Local;
19621
19622        let mut state = RunState::new(
19623            repo.clone(),
19624            "main".to_owned(),
19625            "deadbeef".to_owned(),
19626            "task".to_owned(),
19627            config,
19628        );
19629        state.candidates = vec![Candidate {
19630            index: 0,
19631            label: 'A',
19632            agent: "alpha".to_owned(),
19633            branch: "does-not-exist".to_owned(),
19634            worktree: repo.clone(),
19635            summary: String::new(),
19636            stat: String::new(),
19637            files: 0,
19638            commits: 0,
19639            empty: false,
19640            failed: None,
19641            verified_noop: None,
19642            duration_ms: 0,
19643            folded: false,
19644        }];
19645        state.tally = Some(Tally {
19646            first_choice: BTreeMap::from([('A', 1)]),
19647            borda: BTreeMap::new(),
19648            winner: 'A',
19649            rankings: 1,
19650            unanimous_initial: true,
19651            deliberated: false,
19652            changed_votes: 0,
19653            unanimous_final: true,
19654            tie_break: None,
19655            judges: 0,
19656            present: 0,
19657            quorum: 0,
19658            met_quorum: true,
19659            uncontested: Some("only candidate A produced a change".to_owned()),
19660        });
19661        state.reviews = vec![ReviewRound {
19662            round: 1,
19663            head: "deadbeef".to_owned(),
19664            verified_head: None,
19665            verified_at: None,
19666            reviews: Vec::new(),
19667            e2e: Vec::new(),
19668            fix: None,
19669            blocking: 0,
19670            answered: 0,
19671            expected: 0,
19672            clean: true,
19673            verify_retried: false,
19674            e2e_deferred: false,
19675            e2e_defer_reason: None,
19676            progressed: false,
19677            vote_split: false,
19678            reconsideration: Vec::new(),
19679            verdict: None,
19680        }];
19681        state.gate = vec![CommandOutcome {
19682            command: "test".to_owned(),
19683            code: Some(0),
19684            output_tail: String::new(),
19685            duration_ms: 0,
19686            resource_blocked: false,
19687        }];
19688        state.gate_ran = true;
19689        // Reached its conclusion already — e.g. `land` closing the PR without
19690        // merging it, which (like the honest `MergeMode::None` path) leaves
19691        // `status` at `Ready`. The recorded outcome is what actually marks
19692        // this node done.
19693        state.status = RunStatus::Ready;
19694        state.merge = Some(MergeOutcome {
19695            mode: MergeMode::Local,
19696            ok: false,
19697            detail: "already concluded".to_owned(),
19698            empty: false,
19699        });
19700
19701        let mut runner = Runner {
19702            state,
19703            roles: ResolvedRoles {
19704                implementers: Vec::new(),
19705                judges: Vec::new(),
19706                reviewers: Vec::new(),
19707                fixer: None,
19708                conductor: conductor(),
19709                implementer_roster: Vec::new(),
19710                judge_roster: Vec::new(),
19711                reviewer_roster: Vec::new(),
19712            },
19713            sem: Arc::new(Semaphore::new(1)),
19714            pause: Pause::new(),
19715            interrupt: Pause::new(),
19716        };
19717
19718        runner.merge().await.expect("merge");
19719
19720        assert_eq!(
19721            runner.state.status,
19722            RunStatus::Ready,
19723            "a concluded run's status must not change on reentry"
19724        );
19725        assert_eq!(
19726            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19727            Some("already concluded"),
19728            "merge must not run again once the node already recorded an outcome"
19729        );
19730    }
19731
19732    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19733    /// when its last attempt was resource-blocked (the shared build cache
19734    /// could not be acquired or confirmed fresh in time - see
19735    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19736    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19737    /// a run merge a tree the gate never actually checked - exactly the case
19738    /// a contended cache produces on every retry until it clears. `merge`
19739    /// must refuse until `gate` has actually recorded an attempt.
19740    #[tokio::test]
19741    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19742        let tmp = tempfile::tempdir().expect("tempdir");
19743        let repo = tmp.path().join("repo");
19744        std::fs::create_dir_all(&repo).unwrap();
19745        init_repo(&repo);
19746
19747        let mut config = Config::default();
19748        config.merge.mode = MergeMode::Local;
19749
19750        let mut state = RunState::new(
19751            repo.clone(),
19752            "main".to_owned(),
19753            "deadbeef".to_owned(),
19754            "task".to_owned(),
19755            config,
19756        );
19757        state.candidates = vec![Candidate {
19758            index: 0,
19759            label: 'A',
19760            agent: "alpha".to_owned(),
19761            branch: "does-not-exist".to_owned(),
19762            worktree: repo.clone(),
19763            summary: String::new(),
19764            stat: String::new(),
19765            files: 0,
19766            commits: 0,
19767            empty: false,
19768            failed: None,
19769            verified_noop: None,
19770            duration_ms: 0,
19771            folded: false,
19772        }];
19773        state.tally = Some(Tally {
19774            first_choice: BTreeMap::from([('A', 1)]),
19775            borda: BTreeMap::new(),
19776            winner: 'A',
19777            rankings: 1,
19778            unanimous_initial: true,
19779            deliberated: false,
19780            changed_votes: 0,
19781            unanimous_final: true,
19782            tie_break: None,
19783            judges: 0,
19784            present: 0,
19785            quorum: 0,
19786            met_quorum: true,
19787            uncontested: Some("only candidate A produced a change".to_owned()),
19788        });
19789        state.reviews = vec![ReviewRound {
19790            round: 1,
19791            head: "deadbeef".to_owned(),
19792            verified_head: None,
19793            verified_at: None,
19794            reviews: Vec::new(),
19795            e2e: Vec::new(),
19796            fix: None,
19797            blocking: 0,
19798            answered: 0,
19799            expected: 0,
19800            clean: true,
19801            verify_retried: false,
19802            e2e_deferred: false,
19803            e2e_defer_reason: None,
19804            progressed: false,
19805            vote_split: false,
19806            reconsideration: Vec::new(),
19807            verdict: None,
19808        }];
19809        // The point: `gate` has not recorded anything yet.
19810        state.gate = Vec::new();
19811        state.gate_ran = false;
19812        state.status = RunStatus::Gating;
19813
19814        let mut runner = Runner {
19815            state,
19816            roles: ResolvedRoles {
19817                implementers: Vec::new(),
19818                judges: Vec::new(),
19819                reviewers: Vec::new(),
19820                fixer: None,
19821                conductor: conductor(),
19822                implementer_roster: Vec::new(),
19823                judge_roster: Vec::new(),
19824                reviewer_roster: Vec::new(),
19825            },
19826            sem: Arc::new(Semaphore::new(1)),
19827            pause: Pause::new(),
19828            interrupt: Pause::new(),
19829        };
19830
19831        runner.merge().await.expect("merge");
19832
19833        assert!(
19834            runner.state.merge.is_none(),
19835            "an empty gate must never be read as a passing one: {:?}",
19836            runner.state.merge
19837        );
19838    }
19839
19840    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19841    /// commands configured and `merge.mode` is `none` (a review-only run).
19842    /// `gate` must still record a real attempt — zero commands, vacuously
19843    /// passed — rather than leaving `state.gate` empty in a way `merge`
19844    /// cannot tell apart from "never ran"; otherwise the run reaches
19845    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19846    #[tokio::test]
19847    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19848        let tmp = tempfile::tempdir().expect("tempdir");
19849        let repo = tmp.path().join("repo");
19850        std::fs::create_dir_all(&repo).unwrap();
19851        init_repo(&repo);
19852
19853        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19854        let config = Config::default();
19855
19856        let mut state = RunState::new(
19857            repo.clone(),
19858            "main".to_owned(),
19859            "deadbeef".to_owned(),
19860            "task".to_owned(),
19861            config,
19862        );
19863        state.candidates = vec![Candidate {
19864            index: 0,
19865            label: 'A',
19866            agent: "alpha".to_owned(),
19867            branch: "does-not-exist".to_owned(),
19868            worktree: repo.clone(),
19869            summary: String::new(),
19870            stat: String::new(),
19871            files: 0,
19872            commits: 0,
19873            empty: false,
19874            failed: None,
19875            verified_noop: None,
19876            duration_ms: 0,
19877            folded: false,
19878        }];
19879        state.tally = Some(Tally {
19880            first_choice: BTreeMap::from([('A', 1)]),
19881            borda: BTreeMap::new(),
19882            winner: 'A',
19883            rankings: 1,
19884            unanimous_initial: true,
19885            deliberated: false,
19886            changed_votes: 0,
19887            unanimous_final: true,
19888            tie_break: None,
19889            judges: 0,
19890            present: 0,
19891            quorum: 0,
19892            met_quorum: true,
19893            uncontested: Some("only candidate A produced a change".to_owned()),
19894        });
19895        state.reviews = vec![ReviewRound {
19896            round: 1,
19897            head: "deadbeef".to_owned(),
19898            verified_head: None,
19899            verified_at: None,
19900            reviews: Vec::new(),
19901            e2e: Vec::new(),
19902            fix: None,
19903            blocking: 0,
19904            answered: 0,
19905            expected: 0,
19906            clean: true,
19907            verify_retried: false,
19908            e2e_deferred: false,
19909            e2e_defer_reason: None,
19910            progressed: false,
19911            vote_split: false,
19912            reconsideration: Vec::new(),
19913            verdict: None,
19914        }];
19915
19916        let mut runner = Runner {
19917            state,
19918            roles: ResolvedRoles {
19919                implementers: Vec::new(),
19920                judges: Vec::new(),
19921                reviewers: Vec::new(),
19922                fixer: None,
19923                conductor: conductor(),
19924                implementer_roster: Vec::new(),
19925                judge_roster: Vec::new(),
19926                reviewer_roster: Vec::new(),
19927            },
19928            sem: Arc::new(Semaphore::new(1)),
19929            pause: Pause::new(),
19930            interrupt: Pause::new(),
19931        };
19932
19933        runner.gate().await.expect("gate");
19934        assert!(
19935            runner.state.gate_ran,
19936            "zero configured commands is still a real attempt, not an unrun gate"
19937        );
19938        assert!(runner.state.gate.is_empty());
19939        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19940        assert_ne!(
19941            runner.state.status,
19942            RunStatus::Blocked,
19943            "a gate with nothing to check must not read as failed"
19944        );
19945
19946        runner.merge().await.expect("merge");
19947        assert_eq!(
19948            runner.state.status,
19949            RunStatus::Ready,
19950            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19951        );
19952    }
19953
19954    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19955    /// `verify.gate` (so the e2e leg and the final gate never build against
19956    /// different directories). With zero `verify.gate` commands but a
19957    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19958    /// that lease before discovering it had nothing to run - so a repo with
19959    /// no gate commands could come back `resource_blocked` (and therefore
19960    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19961    /// cache, exactly the contention this run's own zero commands could
19962    /// never have touched. `gate` must recognise there is nothing to check
19963    /// before it ever asks for the lease.
19964    #[tokio::test]
19965    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19966        crate::run::pin_test_home();
19967        let home = crate::run::home();
19968
19969        let tmp = tempfile::tempdir().expect("tempdir");
19970        let repo = tmp.path().join("repo");
19971        std::fs::create_dir_all(&repo).unwrap();
19972        init_repo(&repo);
19973        // Unique to this test, so holding its lease cannot collide with
19974        // another test sharing the same process-wide `home`.
19975        let cache_dir = tmp.path().join("target");
19976
19977        let mut config = Config::default();
19978        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19979        // `verify.gate` stays empty (the default). Any accidental cache
19980        // acquisition must report contention immediately, regardless of how
19981        // long unrelated Git subprocesses take under parallel test load.
19982        config.graph.timeout_verify = Some(2);
19983        config.graph.timeout_resource_wait = 0;
19984
19985        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19986        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19987            .expect("no io error acquiring directly")
19988        {
19989            crate::cache::AcquireOutcome::Acquired(g) => g,
19990            crate::cache::AcquireOutcome::Busy(b) => {
19991                panic!("expected the direct acquire to win the lease first: {b:?}")
19992            }
19993        };
19994
19995        let mut state = RunState::new(
19996            repo.clone(),
19997            "main".to_owned(),
19998            "deadbeef".to_owned(),
19999            "task".to_owned(),
20000            config,
20001        );
20002        state.candidates = vec![Candidate {
20003            index: 0,
20004            label: 'A',
20005            agent: "alpha".to_owned(),
20006            branch: "does-not-exist".to_owned(),
20007            worktree: repo.clone(),
20008            summary: String::new(),
20009            stat: String::new(),
20010            files: 0,
20011            commits: 0,
20012            empty: false,
20013            failed: None,
20014            verified_noop: None,
20015            duration_ms: 0,
20016            folded: false,
20017        }];
20018        state.tally = Some(Tally {
20019            first_choice: BTreeMap::from([('A', 1)]),
20020            borda: BTreeMap::new(),
20021            winner: 'A',
20022            rankings: 1,
20023            unanimous_initial: true,
20024            deliberated: false,
20025            changed_votes: 0,
20026            unanimous_final: true,
20027            tie_break: None,
20028            judges: 0,
20029            present: 0,
20030            quorum: 0,
20031            met_quorum: true,
20032            uncontested: Some("only candidate A produced a change".to_owned()),
20033        });
20034        state.reviews = vec![ReviewRound {
20035            round: 1,
20036            head: "deadbeef".to_owned(),
20037            verified_head: None,
20038            verified_at: None,
20039            reviews: Vec::new(),
20040            e2e: Vec::new(),
20041            fix: None,
20042            blocking: 0,
20043            answered: 0,
20044            expected: 0,
20045            clean: true,
20046            verify_retried: false,
20047            e2e_deferred: false,
20048            e2e_defer_reason: None,
20049            progressed: false,
20050            vote_split: false,
20051            reconsideration: Vec::new(),
20052            verdict: None,
20053        }];
20054
20055        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20056        state.reviews[0].e2e = vec![CommandOutcome {
20057            command: state.config.verify.e2e[0].clone(),
20058            code: Some(0),
20059            output_tail: String::new(),
20060            duration_ms: 0,
20061            resource_blocked: false,
20062        }];
20063        let mut runner = Runner {
20064            state,
20065            roles: ResolvedRoles {
20066                implementers: Vec::new(),
20067                judges: Vec::new(),
20068                reviewers: Vec::new(),
20069                fixer: None,
20070                conductor: conductor(),
20071                implementer_roster: Vec::new(),
20072                judge_roster: Vec::new(),
20073                reviewer_roster: Vec::new(),
20074            },
20075            sem: Arc::new(Semaphore::new(1)),
20076            pause: Pause::new(),
20077            interrupt: Pause::new(),
20078        };
20079
20080        runner.gate().await.expect("gate");
20081        assert!(runner.state.resource_block.is_none());
20082        assert!(
20083            !runner
20084                .state
20085                .events
20086                .iter()
20087                .any(|event| { event.message.contains("waiting for the shared build cache") }),
20088            "a gate with nothing to run must never request the held cache lease"
20089        );
20090        assert!(
20091            runner.state.gate_ran,
20092            "zero commands is still a real, immediate attempt"
20093        );
20094        assert!(runner.state.gate.is_empty());
20095        assert_ne!(
20096            runner.state.status,
20097            RunStatus::Blocked,
20098            "must not read as resource-blocked on a lease it never asked for"
20099        );
20100    }
20101
20102    /// A separate process carries a live worker's environment throughout all
20103    /// verification entry points; the parallel harness never changes its env.
20104    #[cfg(unix)]
20105    #[tokio::test]
20106    async fn verification_is_standalone_inside_a_live_worker() {
20107        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20108        if std::env::var_os(CHILD).is_none() {
20109            let home = tempfile::tempdir().unwrap();
20110            let generation = crate::worker::pin_current(home.path()).unwrap();
20111            let output = std::process::Command::new(std::env::current_exe().unwrap())
20112                .args([
20113                    "--exact",
20114                    "graph::tests::verification_is_standalone_inside_a_live_worker",
20115                    "--nocapture",
20116                ])
20117                .env(CHILD, "1")
20118                .env("MAGI_HOME", home.path())
20119                .env("MAGI_WORKER_HOME", home.path())
20120                .env("MAGI_WORKER_ID", "abcdef")
20121                .env("MAGI_WORKER_GENERATION", &generation.id)
20122                .env("MAGI_WORKER_EXE", &generation.executable)
20123                .env("MAGI_RUN", "verification-parent")
20124                .env("MAGI_NODE", "gate")
20125                .env("MAGI_SEAT", "worker-fixture")
20126                .env("MAGI_PROCESS_OWNER", "fixture-owner")
20127                .env("MAGI_NO_AUTOUPDATE", "0")
20128                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20129                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20130                .env(
20131                    "VERIFY_EXPECT_TOOLCHAIN",
20132                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20133                )
20134                .env(
20135                    "VERIFY_EXPECT_CACHE",
20136                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20137                )
20138                .output()
20139                .unwrap();
20140            assert!(
20141                output.status.success()
20142                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20143                "{}\n{}",
20144                String::from_utf8_lossy(&output.stdout),
20145                String::from_utf8_lossy(&output.stderr)
20146            );
20147            return;
20148        }
20149        let home = crate::run::home();
20150        let directory = home.join("workers/jobs/abcdef");
20151        std::fs::create_dir_all(&directory).unwrap();
20152        let job = crate::worker::Job {
20153            id: "abcdef".into(),
20154            sequence: 1,
20155            kind: "task".into(),
20156            key: "task:verification".into(),
20157            payload: serde_json::json!({}),
20158            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20159            version: Some(env!("CARGO_PKG_VERSION").into()),
20160            status: crate::worker::Status::Running,
20161            result: None,
20162            pid: Some(std::process::id()),
20163            created_at: 0,
20164            finished_at: None,
20165        };
20166        std::fs::write(
20167            directory.join("job.json"),
20168            serde_json::to_vec(&job).unwrap(),
20169        )
20170        .unwrap();
20171        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20172        let assert_worker = || {
20173            assert_eq!(
20174                crate::worker::current_job(&home).unwrap().unwrap().id,
20175                job.id
20176            );
20177            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20178            assert_eq!(
20179                crate::worker::helper_executable(&home).unwrap(),
20180                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20181            );
20182            assert!(
20183                crate::updater::Checker::new(&crate::config::Update {
20184                    mode: crate::config::UpdateMode::Notify,
20185                    interval: None,
20186                    auto_restart: false,
20187                })
20188                .is_none(),
20189                "the driver remains pinned, even while tests are standalone"
20190            );
20191        };
20192        assert_worker();
20193        let repo = home.join("repo");
20194        std::fs::create_dir_all(&repo).unwrap();
20195        init_repo(&repo);
20196        // Keep storage, seat/control context, containment and tool lookup.
20197        // No inherited values (or credentials) are printed on success.
20198        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20199            test -z "${MAGI_WORKER_GENERATION+x}" &&
20200            test -z "${MAGI_WORKER_HOME+x}" &&
20201            test -z "${MAGI_WORKER_EXE+x}" &&
20202            test "$MAGI_HOME/repo" -ef . &&
20203            test "$MAGI_RUN" = verification-parent &&
20204            test "$MAGI_NODE" = gate &&
20205            test "$MAGI_SEAT" = worker-fixture &&
20206            test -n "$MAGI_PROCESS_OWNER" &&
20207            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20208            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20209            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20210            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20211        // Negative control: an ordinary child inherits the contamination.
20212        assert!(
20213            !std::process::Command::new("sh")
20214                .args(["-c", probe])
20215                .current_dir(&repo)
20216                .status()
20217                .unwrap()
20218                .success()
20219        );
20220        let unrelated_home = tempfile::tempdir().unwrap();
20221        assert!(
20222            crate::worker::current_job(unrelated_home.path()).is_err(),
20223            "an inherited worker id cannot be resolved in a unit fixture's home"
20224        );
20225        let mut runner = runner_at(RunStatus::Gating);
20226        runner.state.repo = repo.clone();
20227        runner.state.config.disk.min_free_bytes = 0;
20228        runner.state.config.coordination = crate::config::test_coordination();
20229        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20230        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20231        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20232        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20233        let winner = Candidate {
20234            index: 0,
20235            label: 'A',
20236            agent: "fixture".into(),
20237            branch: "main".into(),
20238            worktree: repo.clone(),
20239            summary: String::new(),
20240            stat: String::new(),
20241            files: 0,
20242            commits: 0,
20243            empty: false,
20244            failed: None,
20245            verified_noop: None,
20246            duration_ms: 0,
20247            folded: false,
20248        };
20249        let (e2e, retried, pids) = run_e2e_with_retry(
20250            &mut runner.state,
20251            &["sh".into(), "-c".into()],
20252            &[format!("{probe} && printf initial")],
20253            &repo,
20254            Duration::from_secs(10),
20255            "initial",
20256        )
20257        .await;
20258        assert!(!retried && pids.is_empty());
20259        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20260        assert_eq!(e2e[0].output_tail, "initial");
20261        let gate = runner.run_gate(&winner).await.unwrap();
20262        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20263        // An interrupted repair must recheck e2e from saved state. Landing
20264        // reuses that state and must run both pre-gate and gate again.
20265        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20266            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20267            passed: false,
20268            outcomes: Vec::new(),
20269        });
20270        runner.state.save().unwrap();
20271        let mut resumed = RunState::load(&runner.state.id).unwrap();
20272        let wait = crate::repo_lock::WaitPolicy::default();
20273        assert!(matches!(
20274            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20275                .await
20276                .unwrap(),
20277            GateFix::Retry
20278        ));
20279        let saved = RunState::load(&resumed.id).unwrap();
20280        let repair = saved.gate_repair_e2e.unwrap();
20281        assert!(repair.passed);
20282        assert_eq!(repair.outcomes.len(), 1);
20283        assert!(repair.outcomes[0].ok());
20284        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20285            .await
20286            .unwrap();
20287        assert_eq!(landing.len(), 1);
20288        assert!(landing[0].ok(), "{landing:?}");
20289        assert_eq!(
20290            std::fs::read_to_string(repo.join("phases")).unwrap(),
20291            "pre\ngate\ne2e\npre\ngate\n"
20292        );
20293        assert_worker();
20294    }
20295
20296    #[cfg(unix)]
20297    #[tokio::test]
20298    async fn verification_keeps_the_panic_and_the_full_output() {
20299        crate::run::pin_test_home();
20300        let tmp = tempfile::tempdir().unwrap();
20301        let mut config = Config::default();
20302        config.disk.min_free_bytes = 0;
20303        let mut state = RunState::new(
20304            tmp.path().to_owned(),
20305            "main".into(),
20306            "base".into(),
20307            "verification evidence".into(),
20308            config,
20309        );
20310        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20311                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20312                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20313        let mut names = Vec::new();
20314        for _ in 0..2 {
20315            let (outcomes, _) = run_commands(
20316                &mut state,
20317                "gate",
20318                "gate",
20319                0,
20320                &["sh".into(), "-c".into()],
20321                &[script.into()],
20322                tmp.path(),
20323                Duration::from_secs(60),
20324            )
20325            .await;
20326            let tail = &outcomes[0].output_tail;
20327            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20328            assert!(tail.contains("boom assertion"));
20329            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20330            let line = tail.lines().next().unwrap();
20331            names.push(line.to_owned());
20332        }
20333        assert_ne!(names[0], names[1], "the second run must not overwrite");
20334        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20335        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20336        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20337
20338        // Storage failures are evidence too; preserve the command's verdict.
20339        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20340        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20341        let (outcomes, _) = run_commands(
20342            &mut state,
20343            "gate",
20344            "gate",
20345            0,
20346            &["sh".into(), "-c".into()],
20347            &["echo failure-output; exit 7".into()],
20348            tmp.path(),
20349            Duration::from_secs(10),
20350        )
20351        .await;
20352        assert_eq!(outcomes[0].code, Some(7));
20353        assert!(state.events.iter().any(|e| {
20354            e.node == "gate" && e.message.contains("could not save the full gate output:")
20355        }));
20356    }
20357
20358    #[cfg(unix)]
20359    #[tokio::test]
20360    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20361        crate::run::pin_test_home();
20362        let tmp = tempfile::tempdir().unwrap();
20363        let mut config = Config::default();
20364        config.disk.min_free_bytes = 0;
20365        let mut state = RunState::new(
20366            tmp.path().to_owned(),
20367            "main".into(),
20368            "base".into(),
20369            "verification evidence".into(),
20370            config,
20371        );
20372        let (outcomes, pids) = run_commands(
20373            &mut state,
20374            "gate",
20375            "gate",
20376            1,
20377            &["sh".into(), "-c".into()],
20378            &[
20379                "printf before-timeout; exec sleep 30".into(),
20380                "kill -TERM $$".into(),
20381                "exit 7".into(),
20382                "printf success".into(),
20383            ],
20384            tmp.path(),
20385            Duration::from_secs(1),
20386        )
20387        .await;
20388        wait_for_timed_out_children_to_die(&pids).await;
20389        assert_eq!(
20390            pids.len(),
20391            1,
20392            "unleased timeout retains its death-check PID"
20393        );
20394        assert_eq!(outcomes.len(), 4);
20395        assert_eq!(outcomes[0].code, None);
20396        assert!(outcomes[0].output_tail.contains("before-timeout"));
20397        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20398        assert_eq!(outcomes[1].code, None);
20399        assert!(outcomes[1].output_tail.contains("command terminated:"));
20400        assert!(!outcomes[1].output_tail.contains("timed out"));
20401        assert_eq!(outcomes[2].code, Some(7));
20402        assert!(!outcomes[2].output_tail.contains("timed out"));
20403        assert_eq!(outcomes[3].code, Some(0));
20404        assert_eq!(outcomes[3].output_tail, "success");
20405        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20406    }
20407
20408    /// The addendum's second gap: a `verify.gate` command running for real
20409    /// wall-clock time had nothing at all to show for it in `active` before
20410    /// `run_commands` learned to record it — a run could sit in `Gating` for
20411    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20412    /// what was actually happening. Proven with a genuinely still-running
20413    /// command, not just a before/after check on the final state: a poller
20414    /// task reads the same `run.json` `gate()` is writing, the same way the
20415    /// phone or `magi show` would, while the shell command is still blocked
20416    /// on its own release marker.
20417    #[tokio::test]
20418    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20419        crate::run::pin_test_home();
20420
20421        let tmp = tempfile::tempdir().expect("tempdir");
20422        let repo = tmp.path().join("repo");
20423        std::fs::create_dir_all(&repo).unwrap();
20424        init_repo(&repo);
20425
20426        let mut config = Config::default();
20427        config.disk.min_free_bytes = 0;
20428        config.verify.gate = vec![
20429            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20430             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20431                .to_owned(),
20432        ];
20433
20434        let mut state = RunState::new(
20435            repo.clone(),
20436            "main".to_owned(),
20437            "deadbeef".to_owned(),
20438            "task".to_owned(),
20439            config,
20440        );
20441        let run_id = state.id.clone();
20442        state.candidates = vec![Candidate {
20443            index: 0,
20444            label: 'A',
20445            agent: "alpha".to_owned(),
20446            branch: "does-not-exist".to_owned(),
20447            worktree: repo.clone(),
20448            summary: String::new(),
20449            stat: String::new(),
20450            files: 0,
20451            commits: 0,
20452            empty: false,
20453            failed: None,
20454            verified_noop: None,
20455            duration_ms: 0,
20456            folded: false,
20457        }];
20458        state.tally = Some(Tally {
20459            first_choice: BTreeMap::from([('A', 1)]),
20460            borda: BTreeMap::new(),
20461            winner: 'A',
20462            rankings: 1,
20463            unanimous_initial: true,
20464            deliberated: false,
20465            changed_votes: 0,
20466            unanimous_final: true,
20467            tie_break: None,
20468            judges: 0,
20469            present: 0,
20470            quorum: 0,
20471            met_quorum: true,
20472            uncontested: Some("only candidate A produced a change".to_owned()),
20473        });
20474        state.reviews = vec![ReviewRound {
20475            round: 1,
20476            head: "deadbeef".to_owned(),
20477            verified_head: None,
20478            verified_at: None,
20479            reviews: Vec::new(),
20480            e2e: Vec::new(),
20481            fix: None,
20482            blocking: 0,
20483            answered: 0,
20484            expected: 0,
20485            clean: true,
20486            verify_retried: false,
20487            e2e_deferred: false,
20488            e2e_defer_reason: None,
20489            progressed: false,
20490            vote_split: false,
20491            reconsideration: Vec::new(),
20492            verdict: None,
20493        }];
20494
20495        let mut runner = Runner {
20496            state,
20497            roles: ResolvedRoles {
20498                implementers: Vec::new(),
20499                judges: Vec::new(),
20500                reviewers: Vec::new(),
20501                fixer: None,
20502                conductor: conductor(),
20503                implementer_roster: Vec::new(),
20504                judge_roster: Vec::new(),
20505                reviewer_roster: Vec::new(),
20506            },
20507            sem: Arc::new(Semaphore::new(1)),
20508            pause: Pause::new(),
20509            interrupt: Pause::new(),
20510        };
20511
20512        let started_marker = repo.join("started.marker");
20513        let release_marker = repo.join("release.marker");
20514        let poller = tokio::spawn(async move {
20515            // Bounded so a regression that never records the task entry
20516            // fails this test in seconds instead of hanging the suite —
20517            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20518            // does_not_cut_it_short` uses for the same reason.
20519            for _ in 0..100 {
20520                if started_marker.exists()
20521                    && let Ok(s) = crate::run::RunState::load(&run_id)
20522                    && let Some(a) = s.active.get("gate")
20523                {
20524                    std::fs::write(&release_marker, b"go").expect("release marker");
20525                    return Some(a.clone());
20526                }
20527                tokio::time::sleep(Duration::from_millis(50)).await;
20528            }
20529            None
20530        });
20531
20532        runner.gate().await.expect("gate");
20533        let captured = poller.await.expect("poller task");
20534        let captured = captured.expect(
20535            "the poller never saw a `gate` task entry in run.json while the command was \
20536             still blocked on its own release marker",
20537        );
20538
20539        assert_eq!(captured.task.as_deref(), Some("gate"));
20540        assert_eq!(captured.node, "gate");
20541        assert_eq!(captured.index, Some(1));
20542        assert_eq!(captured.total, Some(1));
20543        assert!(
20544            captured
20545                .command
20546                .as_deref()
20547                .is_some_and(|c| c.contains("started.marker")),
20548            "{captured:?}"
20549        );
20550
20551        assert!(
20552            runner.state.active.is_empty(),
20553            "the entry must be cleared once the command actually finished: {:?}",
20554            runner.state.active
20555        );
20556        assert!(runner.state.gate_ran);
20557        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20558    }
20559
20560    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20561    /// record for `land`; one with only a Minor, or no reject, leaves none.
20562    #[tokio::test]
20563    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20564        use crate::verdict::{Finding, ReviewVote, Severity};
20565        crate::run::pin_test_home();
20566        let tmp = tempfile::tempdir().expect("tempdir");
20567        let repo = tmp.path().join("repo");
20568        std::fs::create_dir_all(&repo).unwrap();
20569        init_repo(&repo);
20570
20571        for (severity, vote, expect) in [
20572            (Severity::Major, ReviewVote::Reject, true),
20573            (Severity::Minor, ReviewVote::Reject, false),
20574            (Severity::Major, ReviewVote::Approve, false),
20575        ] {
20576            let mut round = review_round(false, 1, 1, 1, false, true);
20577            round.reviews = vec![ReviewRecord {
20578                reviewer: 1,
20579                agent: "alpha".to_owned(),
20580                summary: String::new(),
20581                findings: vec![Finding {
20582                    id: "R1-1-1".to_owned(),
20583                    severity,
20584                    file: None,
20585                    line: None,
20586                    title: "t".to_owned(),
20587                    detail: String::new(),
20588                }],
20589                vote: Some(vote),
20590                failed: None,
20591                duration_ms: 0,
20592                attempts: 0,
20593            }];
20594            let mut state = RunState::new(
20595                repo.clone(),
20596                "main".to_owned(),
20597                "deadbeef".to_owned(),
20598                "task".to_owned(),
20599                Config::default(),
20600            );
20601            state.reviews = vec![round];
20602            let mut runner = Runner {
20603                state,
20604                roles: ResolvedRoles {
20605                    implementers: Vec::new(),
20606                    judges: Vec::new(),
20607                    reviewers: Vec::new(),
20608                    fixer: None,
20609                    conductor: conductor(),
20610                    implementer_roster: Vec::new(),
20611                    judge_roster: Vec::new(),
20612                    reviewer_roster: Vec::new(),
20613                },
20614                sem: Arc::new(Semaphore::new(1)),
20615                pause: Pause::new(),
20616                interrupt: Pause::new(),
20617            };
20618            let shell = runner.state.config.shell();
20619            runner
20620                .stop_reviewing("round budget spent", &shell, &repo)
20621                .await
20622                .expect("stop_reviewing");
20623            assert_eq!(runner.state.status, RunStatus::Gating);
20624            assert_eq!(
20625                runner.state.contested_handoff.is_some(),
20626                expect,
20627                "{severity:?} + {vote:?}"
20628            );
20629        }
20630    }
20631
20632    /// The shape the incident this whole fix responds to actually had: the
20633    /// round budget spent, the last round's own e2e blocked on the shared
20634    /// build cache (held here by a live pid — this test process — exactly
20635    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20636    /// without forking a process). `stop_reviewing` must retry it — not
20637    /// silently leave the round looking untouched (the catch-up-only half of
20638    /// the bug), and not read the contention as a red `e2e` and block the
20639    /// run on it (the other half). Called directly, the same way
20640    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20641    /// above exercises `gate`, so this never needs a real cargo build to
20642    /// reach: the lease is never released, so `with_cache_lease` never gets
20643    /// past acquiring it into anything that would need a real workspace.
20644    #[tokio::test]
20645    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20646        crate::run::pin_test_home();
20647        let home = crate::run::home();
20648
20649        let tmp = tempfile::tempdir().expect("tempdir");
20650        let repo = tmp.path().join("repo");
20651        std::fs::create_dir_all(&repo).unwrap();
20652        init_repo(&repo);
20653        let head = crate::git::rev_parse(&repo, "HEAD")
20654            .await
20655            .expect("rev-parse");
20656        // Unique to this test, so holding its lease cannot collide with
20657        // another test sharing the same process-wide `home`.
20658        let cache_dir = tmp.path().join("target");
20659
20660        let mut config = Config::default();
20661        config.verify.e2e = vec![format!(
20662            "CARGO_TARGET_DIR='{}' test -f README.md",
20663            cache_dir.display()
20664        )];
20665        config.graph.review_rounds = 1;
20666        // Bounded so a regression that does start waiting fails the test in
20667        // seconds, not hangs it.
20668        config.graph.timeout_verify = Some(2);
20669        config.graph.timeout_resource_wait = 2;
20670
20671        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20672        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20673            .expect("no io error acquiring directly")
20674        {
20675            crate::cache::AcquireOutcome::Acquired(g) => g,
20676            crate::cache::AcquireOutcome::Busy(b) => {
20677                panic!("expected the direct acquire to win the lease first: {b:?}")
20678            }
20679        };
20680
20681        let mut state = RunState::new(
20682            repo.clone(),
20683            "main".to_owned(),
20684            head.clone(),
20685            "task".to_owned(),
20686            config,
20687        );
20688        state.candidates = vec![Candidate {
20689            index: 0,
20690            label: 'A',
20691            agent: "alpha".to_owned(),
20692            branch: "does-not-exist".to_owned(),
20693            worktree: repo.clone(),
20694            summary: String::new(),
20695            stat: String::new(),
20696            files: 0,
20697            commits: 0,
20698            empty: false,
20699            failed: None,
20700            verified_noop: None,
20701            duration_ms: 0,
20702            folded: false,
20703        }];
20704        state.tally = Some(Tally {
20705            first_choice: BTreeMap::from([('A', 1)]),
20706            borda: BTreeMap::new(),
20707            winner: 'A',
20708            rankings: 1,
20709            unanimous_initial: true,
20710            deliberated: false,
20711            changed_votes: 0,
20712            unanimous_final: true,
20713            tie_break: None,
20714            judges: 0,
20715            present: 0,
20716            quorum: 0,
20717            met_quorum: true,
20718            uncontested: Some("only candidate A produced a change".to_owned()),
20719        });
20720        // The round budget's last round, deferred: `needs_catchup_run`'s
20721        // other trigger. `stop_reviewing`'s retry machinery must treat this
20722        // exactly like a resource-blocked attempt once it actually runs.
20723        state.reviews = vec![ReviewRound {
20724            round: 1,
20725            head: head.clone(),
20726            verified_head: None,
20727            verified_at: None,
20728            reviews: Vec::new(),
20729            e2e: Vec::new(),
20730            fix: None,
20731            blocking: 1,
20732            answered: 1,
20733            expected: 1,
20734            clean: false,
20735            verify_retried: false,
20736            e2e_deferred: true,
20737            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20738            progressed: false,
20739            vote_split: false,
20740            reconsideration: Vec::new(),
20741            verdict: None,
20742        }];
20743
20744        let mut runner = Runner {
20745            state,
20746            roles: ResolvedRoles {
20747                implementers: Vec::new(),
20748                judges: Vec::new(),
20749                reviewers: Vec::new(),
20750                fixer: None,
20751                conductor: conductor(),
20752                implementer_roster: Vec::new(),
20753                judge_roster: Vec::new(),
20754                reviewer_roster: Vec::new(),
20755            },
20756            sem: Arc::new(Semaphore::new(1)),
20757            pause: Pause::new(),
20758            interrupt: Pause::new(),
20759        };
20760
20761        let shell = runner.state.config.shell();
20762        runner
20763            .stop_reviewing("round budget spent", &shell, &repo)
20764            .await
20765            .expect("stop_reviewing");
20766
20767        let last = runner.state.reviews.last().expect("round record");
20768        assert_eq!(
20769            last.e2e_status(),
20770            E2eStatus::ResourceBlocked,
20771            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20772             failed: {last:?}"
20773        );
20774        assert_eq!(
20775            last.verified_head.as_deref(),
20776            Some(head.as_str()),
20777            "which commit this attempt targeted is known even though nothing finished checking \
20778             it"
20779        );
20780        let first_attempt_at = last
20781            .verified_at
20782            .expect("when this attempt ran is known too");
20783        assert_ne!(
20784            runner.state.status,
20785            RunStatus::Blocked,
20786            "contention is evidence about the machine, not the patch — it must not settle the \
20787             run as blocked: {:?}",
20788            runner.state.status
20789        );
20790        assert!(
20791            !runner
20792                .state
20793                .events
20794                .iter()
20795                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20796            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20797            runner.state.events
20798        );
20799
20800        // The cache is still held: a later reentry must retry the same
20801        // round's verification again — not leave it looking exactly as
20802        // untouched as the first blocked attempt, which is indistinguishable
20803        // from never having tried again at all.
20804        runner
20805            .stop_reviewing("round budget spent", &shell, &repo)
20806            .await
20807            .expect("stop_reviewing retry");
20808        assert_eq!(
20809            runner.state.reviews.len(),
20810            1,
20811            "no new round was started: {:?}",
20812            runner.state.reviews
20813        );
20814        let last = runner.state.reviews.last().expect("round record");
20815        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20816        assert!(
20817            last.verified_at.expect("still known") > first_attempt_at,
20818            "a second reentry must be a fresh attempt, not a stale copy of the first"
20819        );
20820        assert_ne!(runner.state.status, RunStatus::Blocked);
20821
20822        held.release();
20823    }
20824
20825    #[cfg(unix)]
20826    #[tokio::test]
20827    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20828        ask_test_home();
20829        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20830            let tmp = tempfile::tempdir().unwrap();
20831            let repo = tmp.path().join("repo");
20832            std::fs::create_dir(&repo).unwrap();
20833            init_repo(&repo);
20834            let run_git = |args: &[&str]| {
20835                let out = std::process::Command::new("git")
20836                    .args(args)
20837                    .current_dir(&repo)
20838                    .output()
20839                    .unwrap();
20840                assert!(
20841                    out.status.success(),
20842                    "{args:?}: {}",
20843                    String::from_utf8_lossy(&out.stderr)
20844                );
20845            };
20846            let origin = tmp.path().join("origin.git");
20847            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20848            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20849            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20850            run_git(&["add", "spec.rs"]);
20851            run_git(&["commit", "-m", "initial spec"]);
20852            run_git(&["push", "origin", "main"]);
20853            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20854            run_git(&["checkout", "-b", "fix/review"]);
20855            std::fs::write(
20856                repo.join("check.rs"),
20857                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20858            )
20859            .unwrap();
20860            run_git(&["add", "check.rs"]);
20861            run_git(&["commit", "-m", "use spec"]);
20862            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20863            run_git(&["checkout", "main"]);
20864            std::fs::write(
20865                repo.join("spec.rs"),
20866                "struct AgentSpec { max_turn_secs: u64 }\n",
20867            )
20868            .unwrap();
20869            run_git(&["add", "spec.rs"]);
20870            run_git(&["commit", "-m", "extend spec"]);
20871            run_git(&["push", "origin", "main"]);
20872            run_git(&["checkout", "fix/review"]);
20873
20874            let mut runner = runner_at(RunStatus::Reviewing);
20875            runner.state.repo = repo.clone();
20876            runner.state.base_commit = base;
20877            runner.state.pr_head = Some("fix/review".into());
20878            runner.state.config.graph.review_rounds = max_rounds;
20879            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20880            runner.state.config.graph.sessions = false;
20881            runner.state.config.disk.min_free_bytes = 0;
20882            runner.state.config.verify.e2e =
20883                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20884            runner.state.candidates = vec![Candidate {
20885                index: 0,
20886                label: 'A',
20887                agent: "fixture".into(),
20888                branch: "fix/review".into(),
20889                worktree: repo.clone(),
20890                summary: String::new(),
20891                stat: String::new(),
20892                files: 1,
20893                commits: 1,
20894                empty: false,
20895                failed: None,
20896                verified_noop: None,
20897                duration_ms: 0,
20898                folded: false,
20899            }];
20900            runner.state.tally = Some(Tally {
20901                first_choice: BTreeMap::new(),
20902                borda: BTreeMap::new(),
20903                winner: 'A',
20904                rankings: 0,
20905                unanimous_initial: true,
20906                deliberated: false,
20907                changed_votes: 0,
20908                unanimous_final: true,
20909                tie_break: None,
20910                judges: 0,
20911                present: 0,
20912                quorum: 0,
20913                met_quorum: true,
20914                uncontested: Some("review-only".into()),
20915            });
20916            let mut last = review_round(false, 0, 1, 1, false, false);
20917            last.head = reviewed.clone();
20918            last.e2e_deferred = true;
20919            last.e2e[0].resource_blocked = true;
20920            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20921            if max_rounds == 1 {
20922                last.e2e.clear();
20923            }
20924            runner.state.reviews = vec![last];
20925            let calls = tmp.path().join("fix-calls");
20926            let mut fixer = spec("repair");
20927            fixer
20928                .env
20929                .insert("FIX_CALLS".into(), calls.display().to_string());
20930            fixer.command = vec!["sh".into(), "-c".into(), r#"
20931                echo fix >> "$FIX_CALLS"
20932                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20933                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20934                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20935                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20936            "#.into()];
20937            let mut reviewer = spec("reviewer");
20938            reviewer.command = vec![
20939                "sh".into(),
20940                "-c".into(),
20941                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20942                    .into(),
20943            ];
20944            runner.roles.fixer = Some(vec![fixer]);
20945            runner.roles.reviewers = vec![reviewer];
20946
20947            runner.sync_to_base().await.unwrap();
20948            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20949            assert_ne!(rebased, reviewed);
20950            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20951            if reentry {
20952                // Simulate a crash after catch-up evidence was saved but before
20953                // dispatching the fixer. No dedicated pending-repair field exists.
20954                let shell = runner.state.config.shell();
20955                runner
20956                    .stop_unresolved_review("deferred verification", &shell, &repo)
20957                    .await
20958                    .unwrap();
20959                assert!(runner.catchup_repairable());
20960                assert!(!calls.exists());
20961                runner.state = RunState::load(&runner.state.id).unwrap();
20962                assert!(runner.catchup_repairable());
20963            }
20964            runner.review_loop().await.unwrap();
20965            let first = &runner.state.reviews[0];
20966            assert_eq!(first.head, reviewed);
20967            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20968            assert!(
20969                first.verify_retried,
20970                "a reproduced build failure gets exactly one retry: {:?}",
20971                first.e2e
20972            );
20973            assert!(first.e2e[0].output_tail.contains("E0063"));
20974            assert!(!runner.catchup_repairable());
20975            if max_rounds == 2 {
20976                assert_eq!(
20977                    runner.state.status,
20978                    RunStatus::Gating,
20979                    "{}",
20980                    crate::report::run(&runner.state)
20981                );
20982                assert_eq!(runner.state.reviews.len(), 2);
20983                assert!(first.fix.as_ref().unwrap().committed);
20984                assert!(runner.state.reviews[1].clean);
20985                assert_eq!(
20986                    runner
20987                        .state
20988                        .reviews
20989                        .iter()
20990                        .filter(|r| r.fix.is_some())
20991                        .count(),
20992                    1
20993                );
20994                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20995                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20996            } else {
20997                assert_eq!(runner.state.status, RunStatus::Blocked);
20998                assert!(first.fix.is_none());
20999                assert!(!calls.exists());
21000                let recovery = crate::recovery::assess(
21001                    "prior",
21002                    Some(&runner.state),
21003                    &runner.landing_base(),
21004                    crate::recovery::Relation::Diverged,
21005                );
21006                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21007                let reasons = recovery.reasons.join("\n");
21008                assert!(
21009                    reasons.contains(&reviewed) && reasons.contains(&rebased),
21010                    "{reasons}"
21011                );
21012                assert!(
21013                    reasons.contains("base sync to")
21014                        && reasons.contains("additional review is required"),
21015                    "{reasons}"
21016                );
21017            }
21018        }
21019    }
21020
21021    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
21022    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
21023    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
21024    /// own top-of-function fast path (`review_conclusion`) correctly reads
21025    /// this shape as `None` rather than guessing `Blocked`, and the loop's
21026    /// own `for` range is empty once the round budget is spent, so
21027    /// `review_loop` must retry the check itself rather than silently doing
21028    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
21029    /// above exercises directly, but through `review_loop`'s own entry point
21030    /// this time, proving the wiring between the two rather than just the
21031    /// retry logic in isolation.
21032    #[tokio::test]
21033    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21034        crate::run::pin_test_home();
21035        let home = crate::run::home();
21036
21037        let tmp = tempfile::tempdir().expect("tempdir");
21038        let repo = tmp.path().join("repo");
21039        std::fs::create_dir_all(&repo).unwrap();
21040        init_repo(&repo);
21041        let head = crate::git::rev_parse(&repo, "HEAD")
21042            .await
21043            .expect("rev-parse");
21044        let cache_dir = tmp.path().join("target");
21045
21046        let mut config = Config::default();
21047        config.verify.e2e = vec![format!(
21048            "CARGO_TARGET_DIR='{}' test -f README.md",
21049            cache_dir.display()
21050        )];
21051        config.graph.review_rounds = 1;
21052        config.graph.timeout_verify = Some(2);
21053        config.graph.timeout_resource_wait = 2;
21054
21055        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21056        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21057            .expect("no io error acquiring directly")
21058        {
21059            crate::cache::AcquireOutcome::Acquired(g) => g,
21060            crate::cache::AcquireOutcome::Busy(b) => {
21061                panic!("expected the direct acquire to win the lease first: {b:?}")
21062            }
21063        };
21064
21065        let mut state = RunState::new(
21066            repo.clone(),
21067            "main".to_owned(),
21068            head.clone(),
21069            "task".to_owned(),
21070            config,
21071        );
21072        state.candidates = vec![Candidate {
21073            index: 0,
21074            label: 'A',
21075            agent: "alpha".to_owned(),
21076            branch: "does-not-exist".to_owned(),
21077            worktree: repo.clone(),
21078            summary: String::new(),
21079            stat: String::new(),
21080            files: 0,
21081            commits: 0,
21082            empty: false,
21083            failed: None,
21084            verified_noop: None,
21085            duration_ms: 0,
21086            folded: false,
21087        }];
21088        state.tally = Some(Tally {
21089            first_choice: BTreeMap::from([('A', 1)]),
21090            borda: BTreeMap::new(),
21091            winner: 'A',
21092            rankings: 1,
21093            unanimous_initial: true,
21094            deliberated: false,
21095            changed_votes: 0,
21096            unanimous_final: true,
21097            tie_break: None,
21098            judges: 0,
21099            present: 0,
21100            quorum: 0,
21101            met_quorum: true,
21102            uncontested: Some("only candidate A produced a change".to_owned()),
21103        });
21104        // The exact shape a prior process's `stop_reviewing` would have left
21105        // on disk: the round budget's last round, a real attempt already
21106        // made and already resource-blocked.
21107        state.reviews = vec![ReviewRound {
21108            round: 1,
21109            head: head.clone(),
21110            verified_head: Some(head.clone()),
21111            verified_at: Some(jiff::Timestamp::now()),
21112            reviews: Vec::new(),
21113            e2e: vec![CommandOutcome {
21114                command: format!(
21115                    "CARGO_TARGET_DIR='{}' test -f README.md",
21116                    cache_dir.display()
21117                ),
21118                code: None,
21119                output_tail: "waiting for the shared build cache".to_owned(),
21120                duration_ms: 0,
21121                resource_blocked: true,
21122            }],
21123            fix: None,
21124            blocking: 1,
21125            answered: 1,
21126            expected: 1,
21127            clean: false,
21128            verify_retried: false,
21129            e2e_deferred: false,
21130            e2e_defer_reason: None,
21131            progressed: false,
21132            vote_split: false,
21133            reconsideration: Vec::new(),
21134            verdict: None,
21135        }];
21136
21137        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21138        let mut runner = Runner {
21139            state,
21140            roles: ResolvedRoles {
21141                implementers: Vec::new(),
21142                judges: Vec::new(),
21143                reviewers: Vec::new(),
21144                fixer: None,
21145                conductor: conductor(),
21146                implementer_roster: Vec::new(),
21147                judge_roster: Vec::new(),
21148                reviewer_roster: Vec::new(),
21149            },
21150            sem: Arc::new(Semaphore::new(1)),
21151            pause: Pause::new(),
21152            interrupt: Pause::new(),
21153        };
21154
21155        // The lease is still held throughout, so this reentry's own retry is
21156        // also contended — proving `review_loop` actually tried again (not
21157        // that it happened to succeed) is what the timestamp comparison
21158        // below is for.
21159        runner.review_loop().await.expect("review_loop");
21160
21161        assert_eq!(
21162            runner.state.reviews.len(),
21163            1,
21164            "no new round was started on top of the unresolved one: {:?}",
21165            runner.state.reviews
21166        );
21167        let last = &runner.state.reviews[0];
21168        assert_eq!(
21169            last.e2e_status(),
21170            E2eStatus::ResourceBlocked,
21171            "still contended: {last:?}"
21172        );
21173        assert!(
21174            last.verified_at.expect("still known") > first_attempt_at,
21175            "review_loop must have actually retried the check, not left it exactly as found"
21176        );
21177        assert_ne!(
21178            runner.state.status,
21179            RunStatus::Blocked,
21180            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21181            runner.state.status
21182        );
21183
21184        held.release();
21185    }
21186
21187    #[tokio::test]
21188    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21189        crate::run::pin_test_home();
21190        for status in [RunStatus::Blocked, RunStatus::Gating] {
21191            let mut runner = runner_at(status);
21192            runner.state.config.graph.land = true;
21193            runner.state.gate_ran = true;
21194            runner.state.merge = Some(MergeOutcome {
21195                mode: MergeMode::Pr,
21196                ok: false,
21197                detail: "the fixer produced no commit".into(),
21198                empty: false,
21199            });
21200            let url = "https://example.invalid/o/r/pull/1";
21201            runner.state.pr = Some(crate::run::PrRecord {
21202                url: url.into(),
21203                number: 1,
21204                state: "open".into(),
21205                checks: "red".into(),
21206                round: 1,
21207                rounds: 1,
21208                red_at_merge: Vec::new(),
21209            });
21210            runner.state.save().unwrap();
21211            // The nonexistent repository makes observation fail before a forge
21212            // command can run. We still must reach land, rather than silently
21213            // return from the graph with its old gate and merge records.
21214            runner.execute_graph().await.unwrap();
21215            assert_eq!(runner.state.status, RunStatus::Blocked);
21216            assert!(runner.state.land_sync_pending);
21217            assert!(runner.state.events.iter().any(|event|
21218                event.node == "land" && event.message == format!("watching {url}")));
21219            assert_eq!(
21220                runner.state.merge.as_ref().unwrap().detail,
21221                "the fixer produced no commit"
21222            );
21223        }
21224    }
21225
21226    #[tokio::test]
21227    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21228        crate::run::pin_test_home();
21229        let tmp = tempfile::tempdir().expect("tempdir");
21230        let repo = tmp.path().join("repo");
21231        std::fs::create_dir_all(&repo).unwrap();
21232        init_repo(&repo);
21233
21234        let mut config = Config::default();
21235        config.merge.mode = MergeMode::Pr;
21236        config.graph.land = true;
21237        config.graph.land_approval = false;
21238
21239        let mut state = RunState::new(
21240            repo.clone(),
21241            "main".to_owned(),
21242            "deadbeef".to_owned(),
21243            "task".to_owned(),
21244            config,
21245        );
21246        state.candidates = vec![Candidate {
21247            index: 0,
21248            label: 'A',
21249            agent: "alpha".to_owned(),
21250            branch: "does-not-exist".to_owned(),
21251            worktree: repo.clone(),
21252            summary: String::new(),
21253            stat: String::new(),
21254            files: 0,
21255            commits: 0,
21256            empty: false,
21257            failed: None,
21258            verified_noop: None,
21259            duration_ms: 0,
21260            folded: false,
21261        }];
21262        state.tally = Some(Tally {
21263            first_choice: BTreeMap::from([('A', 1)]),
21264            borda: BTreeMap::new(),
21265            winner: 'A',
21266            rankings: 1,
21267            unanimous_initial: true,
21268            deliberated: false,
21269            changed_votes: 0,
21270            unanimous_final: true,
21271            tie_break: None,
21272            judges: 0,
21273            present: 0,
21274            quorum: 0,
21275            met_quorum: true,
21276            uncontested: Some("only candidate A produced a change".to_owned()),
21277        });
21278        state.reviews = vec![ReviewRound {
21279            round: 1,
21280            head: "deadbeef".to_owned(),
21281            verified_head: None,
21282            verified_at: None,
21283            reviews: Vec::new(),
21284            e2e: Vec::new(),
21285            fix: None,
21286            blocking: 0,
21287            answered: 0,
21288            expected: 0,
21289            clean: true,
21290            verify_retried: false,
21291            e2e_deferred: false,
21292            e2e_defer_reason: None,
21293            progressed: false,
21294            vote_split: false,
21295            reconsideration: Vec::new(),
21296            verdict: None,
21297        }];
21298        state.gate = vec![CommandOutcome {
21299            command: "test".to_owned(),
21300            code: Some(0),
21301            output_tail: String::new(),
21302            duration_ms: 0,
21303            resource_blocked: false,
21304        }];
21305        state.gate_ran = true;
21306        // A first pass through `merge` already pushed and opened this pull
21307        // request; `status` is `Landing` because a previous call into `land`
21308        // parked or was interrupted before it reached a terminal outcome.
21309        state.status = RunStatus::Landing;
21310        state.merge = Some(MergeOutcome {
21311            mode: MergeMode::Pr,
21312            ok: true,
21313            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21314            empty: false,
21315        });
21316
21317        // The Landing-resume shortcut calls `run_land` directly rather than
21318        // through `merge`, which is exactly the call site that used to skip
21319        // `settle_questions` - see the fixture below.
21320        ask_test_home();
21321        // A real resume loads a persisted run; driver ownership is checked
21322        // against that record before the Landing shortcut is entered.
21323        state.save().expect("save landing run");
21324        let store = ask::Questions::open();
21325        let q = ask_open_question(&store, &state.id);
21326
21327        let mut runner = Runner {
21328            state,
21329            roles: ResolvedRoles {
21330                implementers: Vec::new(),
21331                judges: Vec::new(),
21332                reviewers: Vec::new(),
21333                fixer: None,
21334                conductor: conductor(),
21335                implementer_roster: Vec::new(),
21336                judge_roster: Vec::new(),
21337                reviewer_roster: Vec::new(),
21338            },
21339            sem: Arc::new(Semaphore::new(1)),
21340            pause: Pause::new(),
21341            interrupt: Pause::new(),
21342        };
21343
21344        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21345        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21346        // exactly because `review_loop` would otherwise clobber the marker
21347        // first.
21348        runner.execute().await.expect("execute");
21349
21350        assert_eq!(
21351            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21352            Some("https://example.invalid/x/y/pull/1"),
21353            "reentry must not push again or open a second pull request over the \
21354             one `land` is already watching"
21355        );
21356        assert_ne!(
21357            runner.state.status,
21358            RunStatus::Landing,
21359            "land could not actually reach the fake pull request, so it must \
21360             have given up rather than left the run silently parked forever"
21361        );
21362        // `land` could not reach the fake pull request, so it gave up into
21363        // `Blocked` - still resumable, so the question must not have been
21364        // swept just because this branch now also calls `settle_questions`.
21365        assert_eq!(runner.state.status, RunStatus::Blocked);
21366        assert!(
21367            store.get(&q.id).unwrap().status.open(),
21368            "Blocked is still alive; settle_questions must have been a no-op here"
21369        );
21370    }
21371
21372    fn state_with_round(round: ReviewRound) -> RunState {
21373        let mut s = RunState::new(
21374            PathBuf::from("/repo"),
21375            "main".to_owned(),
21376            "abc1234".to_owned(),
21377            "add retries".to_owned(),
21378            Config::default(),
21379        );
21380        s.reviews = vec![round];
21381        s
21382    }
21383
21384    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21385        crate::verdict::Finding {
21386            id: id.to_owned(),
21387            severity,
21388            file: None,
21389            line: None,
21390            title: title.to_owned(),
21391            detail: String::new(),
21392        }
21393    }
21394
21395    #[test]
21396    fn pr_body_names_open_findings_and_declined_ones() {
21397        let round = ReviewRound {
21398            round: 2,
21399            head: "deadbee".to_owned(),
21400            verified_head: None,
21401            verified_at: None,
21402            reviews: vec![ReviewRecord {
21403                attempts: 0,
21404                reviewer: 1,
21405                agent: "alpha".to_owned(),
21406                summary: String::new(),
21407                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21408                vote: None,
21409                failed: None,
21410                duration_ms: 0,
21411            }],
21412            e2e: vec![CommandOutcome {
21413                command: "cargo test".to_owned(),
21414                code: Some(0),
21415                output_tail: String::new(),
21416                duration_ms: 0,
21417                resource_blocked: false,
21418            }],
21419            verify_retried: false,
21420            e2e_deferred: false,
21421            e2e_defer_reason: None,
21422            fix: Some(FixRecord {
21423                agent: "alpha".to_owned(),
21424                addressed: Vec::new(),
21425                rejected: vec![crate::verdict::Rejection {
21426                    id: "R1-1-1".to_owned(),
21427                    why: "not reachable from any caller".to_owned(),
21428                }],
21429                notes: String::new(),
21430                committed: true,
21431                failed: None,
21432                duration_ms: 0,
21433                continuation: None,
21434            }),
21435            blocking: 0,
21436            answered: 1,
21437            expected: 1,
21438            clean: false,
21439            progressed: true,
21440            vote_split: false,
21441            reconsideration: Vec::new(),
21442            verdict: None,
21443        };
21444        let state = state_with_round(round);
21445        let body = pr_message(&state, 'A').body;
21446
21447        assert!(body.contains("add retries"), "the task must still be there");
21448        assert!(body.contains("R2-1-1"), "{body}");
21449        assert!(!body.contains("unused import"), "{body}");
21450        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21451        assert!(
21452            !body.contains("not reachable from any caller"),
21453            "the reason it was declined: {body}"
21454        );
21455    }
21456
21457    #[test]
21458    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21459        let round = ReviewRound {
21460            round: 1,
21461            head: "deadbee".to_owned(),
21462            verified_head: None,
21463            verified_at: None,
21464            reviews: vec![ReviewRecord {
21465                attempts: 0,
21466                reviewer: 1,
21467                agent: "alpha".to_owned(),
21468                summary: String::new(),
21469                findings: Vec::new(),
21470                vote: None,
21471                failed: None,
21472                duration_ms: 0,
21473            }],
21474            e2e: Vec::new(),
21475            verify_retried: false,
21476            e2e_deferred: false,
21477            e2e_defer_reason: None,
21478            fix: None,
21479            blocking: 0,
21480            answered: 1,
21481            expected: 1,
21482            clean: true,
21483            progressed: false,
21484            vote_split: false,
21485            reconsideration: Vec::new(),
21486            verdict: None,
21487        };
21488        let state = state_with_round(round);
21489        let body = pr_message(&state, 'A').body;
21490        assert!(!body.contains("Open review findings"), "{body}");
21491        assert!(!body.contains("Declined"), "{body}");
21492    }
21493
21494    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21495        let mut state = RunState::new(
21496            PathBuf::from("/repo"),
21497            "main".to_owned(),
21498            "abc1234".to_owned(),
21499            instruction.to_owned(),
21500            Config::default(),
21501        );
21502        // These fixtures test execution flow, independent of host disk capacity.
21503        state.config.disk.min_free_bytes = 0;
21504        state.candidates.push(Candidate {
21505            index: 0,
21506            label: 'A',
21507            agent: "alpha".to_owned(),
21508            branch: "magi/x/A".to_owned(),
21509            worktree: PathBuf::from("/wt"),
21510            summary: summary.to_owned(),
21511            stat: String::new(),
21512            files: 1,
21513            commits: 1,
21514            empty: false,
21515            failed: None,
21516            verified_noop: None,
21517            folded: false,
21518            duration_ms: 0,
21519        });
21520        state
21521    }
21522
21523    fn review_state(subjects: &[&str]) -> RunState {
21524        let mut state = state_with_summary(
21525            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21526            "",
21527        );
21528        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21529        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21530        state
21531    }
21532
21533    /// A branch rebased onto a main that moved past the recorded
21534    /// `base_commit` is titled from its own first commit, never main's.
21535    #[tokio::test]
21536    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21537        ask_test_home();
21538        let tmp = tempfile::tempdir().unwrap();
21539        let repo = tmp.path().join("repo");
21540        std::fs::create_dir_all(&repo).unwrap();
21541        init_repo(&repo);
21542        let origin = tmp.path().join("origin.git");
21543        let g = |dir: &Path, args: &[&str]| {
21544            let out = std::process::Command::new("git")
21545                .args(args)
21546                .current_dir(dir)
21547                .quiet()
21548                .output()
21549                .expect("spawn git");
21550            assert!(
21551                out.status.success(),
21552                "git {args:?}: {}",
21553                String::from_utf8_lossy(&out.stderr)
21554            );
21555        };
21556        g(
21557            tmp.path(),
21558            &[
21559                "clone",
21560                "--bare",
21561                "-q",
21562                repo.to_str().unwrap(),
21563                origin.to_str().unwrap(),
21564            ],
21565        );
21566        g(
21567            &repo,
21568            &["remote", "add", "origin", origin.to_str().unwrap()],
21569        );
21570        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21571
21572        // Main moves on; the branch is built on top of the new main.
21573        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21574        g(&repo, &["add", "-A"]);
21575        g(
21576            &repo,
21577            &["commit", "-q", "-m", "chore(deps): update a crate"],
21578        );
21579        g(&repo, &["push", "-q", "origin", "main"]);
21580        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21581        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21582        g(&repo, &["add", "-A"]);
21583        g(
21584            &repo,
21585            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21586        );
21587        g(&repo, &["checkout", "-q", "main"]);
21588
21589        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21590        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21591        // Without a readable tracking ref the recorded base's merge base is used.
21592        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21593        assert_eq!(fallback, c1);
21594
21595        let mut state = review_state(&[
21596            "chore(deps): update a crate",
21597            "fix(daemon): apply a chosen action",
21598        ]);
21599        state.repo = repo.clone();
21600        state.base_branch = "main".to_owned();
21601        state.base_commit = c1;
21602        refresh_reviewed_commits(&mut state, "feat/own").await;
21603        assert_eq!(
21604            state.reviewed_commits,
21605            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21606        );
21607        assert_eq!(
21608            review_title(&state).as_deref(),
21609            Some("fix(daemon): apply a chosen action")
21610        );
21611        assert_eq!(
21612            leaked_subjects(&state, "feat/own").await,
21613            Some(vec!["chore(deps): update a crate".to_owned()])
21614        );
21615        // A stale tracking ref is still used when the fetch fails, but the
21616        // leak list is withheld.
21617        state.config.merge.remote = "nowhere".to_owned();
21618        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21619    }
21620
21621    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21622    /// with `origin/main` one commit behind it, and a summarizing agent that
21623    /// prints whatever `reply` holds.
21624    struct SummaryFixture {
21625        _tmp: tempfile::TempDir,
21626        repo: PathBuf,
21627        reply: PathBuf,
21628        runner: Runner,
21629    }
21630
21631    fn summary_fixture() -> SummaryFixture {
21632        ask_test_home();
21633        let tmp = tempfile::tempdir().unwrap();
21634        let repo = tmp.path().join("repo");
21635        std::fs::create_dir_all(&repo).unwrap();
21636        init_repo(&repo);
21637        let origin = tmp.path().join("origin.git");
21638        let g = |args: &[&str]| {
21639            let out = std::process::Command::new("git")
21640                .args(args)
21641                .current_dir(&repo)
21642                .quiet()
21643                .output()
21644                .expect("spawn git");
21645            assert!(
21646                out.status.success(),
21647                "git {args:?}: {}",
21648                String::from_utf8_lossy(&out.stderr)
21649            );
21650        };
21651        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21652        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21653        g(&["push", "-q", "origin", "main"]);
21654        g(&["checkout", "-q", "-b", "feat/own"]);
21655        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21656        g(&["add", "-A"]);
21657        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21658        let reply = tmp.path().join("reply.txt");
21659        std::fs::write(
21660            &reply,
21661            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21662        )
21663        .unwrap();
21664        let mut runner = runner_at(RunStatus::Gating);
21665        let mut state = review_state(&["feat(own): add the own file"]);
21666        state.repo = repo.clone();
21667        state.base_branch = "main".to_owned();
21668        state.candidates[0].branch = "feat/own".to_owned();
21669        state.candidates[0].worktree = repo.clone();
21670        let mut author = spec("alpha");
21671        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21672        author
21673            .env
21674            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21675        state.config.agents = vec![author];
21676        runner.state = state;
21677        SummaryFixture {
21678            _tmp: tmp,
21679            repo,
21680            reply,
21681            runner,
21682        }
21683    }
21684
21685    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21686        let winner = f.runner.state.candidates[0].clone();
21687        f.runner
21688            .guarded_pr_message(&winner, None, true)
21689            .await
21690            .unwrap()
21691            .expect("an English description never parks")
21692    }
21693
21694    fn clean_round_with_minor_findings() -> ReviewRound {
21695        let mut round = passing_round(2, None);
21696        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21697        round.reviews = vec![ReviewRecord {
21698            attempts: 0,
21699            reviewer: 1,
21700            agent: "alpha".into(),
21701            summary: "内部レビュー".into(),
21702            findings: vec![
21703                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21704                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21705            ],
21706            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21707            failed: None,
21708            duration_ms: 0,
21709        }];
21710        round
21711    }
21712
21713    #[tokio::test]
21714    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21715        for review_only in [false, true] {
21716            let mut f = summary_fixture();
21717            if !review_only {
21718                f.runner.state.candidates[0].agent = "alpha".into();
21719            }
21720            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21721            assert!(f.runner.state.reviews[0].clean);
21722            assert!(f.runner.state.reviews[0].fix.is_none());
21723            assert!(f.runner.state.open_findings().is_empty());
21724            assert_eq!(
21725                crate::pr_outcome::required_finding_ids(&f.runner.state),
21726                ["R2-1-1", "R2-1-2"]
21727            );
21728            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21729            let evidence = crate::pr_outcome::summary_evidence(
21730                &f.runner.state,
21731                crate::pr_outcome::FinalHead::At(&head),
21732            );
21733            assert!(
21734                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21735            );
21736            // The ordinary reply omits both IDs: no cached description is adopted.
21737            let omitted = posted(&mut f).await;
21738            assert!(omitted.body.contains("No description"));
21739            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21740            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21741            assert!(f.runner.state.change_summary.is_none());
21742            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21743            let described = posted(&mut f).await;
21744            assert!(described.body.contains("improve documentation for callers"));
21745            assert!(described.body.contains("remaining edge case"));
21746            assert!(described.body.contains("## Open review findings"));
21747            assert!(!described.body.contains("未修正"));
21748        }
21749    }
21750
21751    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21752        let mut f = summary_fixture();
21753        if !review_only {
21754            f.runner.state.candidates[0].agent = "alpha".into();
21755        }
21756        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21757        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21758        fixed
21759            .fix
21760            .as_mut()
21761            .unwrap()
21762            .rejected
21763            .push(crate::verdict::Rejection {
21764                id: "R1-1-1".into(),
21765                why: "呼び出されないコードのため辞退".into(),
21766            });
21767        let mut clean = clean_round_with_minor_findings();
21768        clean.verified_head = Some(head.clone());
21769        f.runner.state.reviews = vec![fixed, clean];
21770        ok_gate(&mut f.runner.state, Some(&head));
21771        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21772        f.runner.state.config.agents.push(scripted("beta", reply));
21773        f.runner.state.config.roles.language_judge =
21774            Some(crate::config::AgentChoice::One("beta".into()));
21775        f
21776    }
21777
21778    #[tokio::test]
21779    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21780        for review_only in [false, true] {
21781            let mut f = rewrite_fixture_with_findings(review_only,
21782                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21783            let m = posted(&mut f).await;
21784            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21785            let expected =
21786                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21787            assert!(m.body.contains(&expected));
21788            assert!(m.body.contains("## Review rounds"));
21789            assert!(
21790                m.body
21791                    .contains("Gate (`verify.gate`): passed on the final commit")
21792            );
21793            assert!(
21794                m.body
21795                    .contains("Tests (`verify.e2e`): passed on the final commit")
21796            );
21797            let c = f.runner.state.change_summary.as_ref().unwrap();
21798            assert!(
21799                m.body
21800                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21801            );
21802            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21803            assert!(crate::pr_outcome::covers_required_findings(
21804                &f.runner.state,
21805                &m.body
21806            ));
21807            assert!(m.body.contains("documentation remains incomplete"));
21808            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21809            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21810            assert!(f.runner.state.github_text.is_none());
21811        }
21812    }
21813
21814    #[tokio::test]
21815    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21816        for review_only in [false, true] {
21817            for reply in [
21818                "TITLE: fix: public description\nA short English description.",
21819                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21820            ] {
21821                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21822                let winner = f.runner.state.candidates[0].clone();
21823                let posted = f
21824                    .runner
21825                    .guarded_pr_message(&winner, None, true)
21826                    .await
21827                    .unwrap();
21828                assert!(posted.is_none());
21829                assert!(f.runner.state.parked);
21830                let withheld = f.runner.state.github_text.as_ref().unwrap();
21831                assert!(
21832                    withheld.rewrite.is_none(),
21833                    "an incomplete rewrite must not be selectable"
21834                );
21835                let q = ask::Questions::open()
21836                    .get(withheld.question.as_ref().unwrap())
21837                    .unwrap();
21838                assert!(
21839                    !q.choices
21840                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21841                );
21842                assert!(
21843                    q.detail.contains("R1-1-1")
21844                        && q.detail.contains("R2-1-1")
21845                        && q.detail.contains("R2-1-2")
21846                );
21847            }
21848        }
21849    }
21850
21851    #[tokio::test]
21852    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21853        for review_only in [false, true] {
21854            let mut f = summary_fixture();
21855            if !review_only {
21856                f.runner.state.candidates[0].agent = "alpha".into();
21857            }
21858            f.runner.state.instruction = "内部タスクの説明".into();
21859            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21860            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21861            f.runner.state.reviews = vec![
21862                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21863                round_with_notes(2, Some("二回目の内部修正報告です。")),
21864                passing_round(3, Some(&head)),
21865            ];
21866            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21867            ok_gate(&mut f.runner.state, Some(&head));
21868            let captured = f.repo.join("public-prompt.txt");
21869            let author = &mut f.runner.state.config.agents[0];
21870            author.command = vec![
21871                "sh".into(),
21872                "-c".into(),
21873                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21874            ];
21875            author.env.insert(
21876                "PROMPT_FILE".into(),
21877                captured.to_string_lossy().into_owned(),
21878            );
21879            let m = posted(&mut f).await;
21880            assert!(m.body.contains("Adds `own.txt`"));
21881            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21882            assert!(m.body.contains("R1-1-1"));
21883            assert!(
21884                m.body
21885                    .contains("Gate (`verify.gate`): passed on the final commit")
21886            );
21887            assert!(
21888                m.body
21889                    .contains("Tests (`verify.e2e`): passed on the final commit")
21890            );
21891            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21892            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21893            let prompt = std::fs::read_to_string(captured).unwrap();
21894            assert!(prompt.contains(&format!(
21895                "git diff {} {head}",
21896                f.runner.state.change_summary.as_ref().unwrap().base
21897            )));
21898            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21899            assert_eq!(
21900                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21901                "二回目の内部修正報告です。"
21902            );
21903        }
21904    }
21905
21906    #[tokio::test]
21907    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21908        for review_only in [false, true] {
21909            let mut f = summary_fixture();
21910            if !review_only {
21911                f.runner.state.candidates[0].agent = "alpha".into();
21912            }
21913            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21914            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21915            std::fs::write(&f.reply, "no usable description").unwrap();
21916            let failed = posted(&mut f).await;
21917            assert!(
21918                failed
21919                    .body
21920                    .contains("No description of what this change does could be generated")
21921            );
21922            assert!(failed.body.is_ascii());
21923            assert!(f.runner.state.github_text.is_none());
21924            assert!(f.runner.state.change_summary.is_none());
21925            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21926            let evidence = crate::pr_outcome::summary_evidence(
21927                &f.runner.state,
21928                crate::pr_outcome::FinalHead::At(&head),
21929            );
21930            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21931            assert!(
21932                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21933                    .exists()
21934            );
21935            f.runner.state =
21936                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21937            std::fs::write(
21938                &f.reply,
21939                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21940            )
21941            .unwrap();
21942            let retried = posted(&mut f).await;
21943            assert!(retried.body.contains("Adds the committed own file."));
21944            assert_eq!(
21945                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21946                "日本語の修正報告"
21947            );
21948        }
21949    }
21950
21951    #[tokio::test]
21952    async fn same_head_verification_and_findings_invalidate_public_prose() {
21953        let mut f = summary_fixture();
21954        posted(&mut f).await;
21955        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21956        let original = f
21957            .runner
21958            .state
21959            .change_summary
21960            .as_ref()
21961            .unwrap()
21962            .evidence
21963            .clone();
21964        ok_gate(&mut f.runner.state, Some(&head));
21965        std::fs::write(
21966            &f.reply,
21967            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21968        )
21969        .unwrap();
21970        let verified = posted(&mut f).await;
21971        assert!(verified.body.contains("now verified"));
21972        assert_ne!(
21973            f.runner.state.change_summary.as_ref().unwrap().evidence,
21974            original
21975        );
21976        let mut round = round_with_notes(1, Some("内部報告"));
21977        round.reviews.push(ReviewRecord {
21978            attempts: 0,
21979            reviewer: 1,
21980            agent: "alpha".into(),
21981            summary: "内部レビュー".into(),
21982            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21983            vote: None,
21984            failed: None,
21985            duration_ms: 0,
21986        });
21987        round
21988            .fix
21989            .as_mut()
21990            .unwrap()
21991            .rejected
21992            .push(crate::verdict::Rejection {
21993                id: "R1-1-1".into(),
21994                why: "呼び出し元から到達できない".into(),
21995            });
21996        f.runner.state.reviews = vec![round];
21997        // A description omitting the required ID is unusable, even with a title.
21998        let omitted = posted(&mut f).await;
21999        assert!(omitted.body.contains("No description"));
22000        assert!(omitted.body.contains("R1-1-1"));
22001        assert!(!omitted.body.contains("到達"));
22002        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22003        let described = posted(&mut f).await;
22004        assert!(described.body.contains("unreachable from callers"));
22005        assert!(described.body.contains("R1-1-1"));
22006        assert!(!described.body.contains("到達"));
22007        let evidence = f
22008            .runner
22009            .state
22010            .change_summary
22011            .as_ref()
22012            .unwrap()
22013            .evidence
22014            .clone();
22015        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22016        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22017        assert!(posted(&mut f).await.body.contains("rationale updated"));
22018        assert_ne!(
22019            f.runner.state.change_summary.as_ref().unwrap().evidence,
22020            evidence
22021        );
22022        // Old serialized summaries have no evidence fingerprint and must regenerate.
22023        f.runner
22024            .state
22025            .change_summary
22026            .as_mut()
22027            .unwrap()
22028            .evidence
22029            .clear();
22030        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22031        assert!(posted(&mut f).await.body.contains("current specification"));
22032    }
22033
22034    #[tokio::test]
22035    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22036        let mut f = summary_fixture();
22037        let m = posted(&mut f).await;
22038        assert_eq!(m.title, "feat(own): add the own file");
22039        assert!(
22040            m.body
22041                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22042            "{}",
22043            m.body
22044        );
22045        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22046        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22047        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22048        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22049        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22050        let kept = f.runner.state.change_summary.as_ref().unwrap();
22051        assert_eq!(
22052            (kept.head.as_str(), kept.base.as_str()),
22053            (head.as_str(), base.as_str())
22054        );
22055        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22056    }
22057
22058    #[tokio::test]
22059    async fn review_only_summary_receives_the_configured_github_language() {
22060        let mut f = summary_fixture();
22061        f.runner.state.config.graph.github_language = "ja".into();
22062        let captured = f.repo.join("summary-prompt.txt");
22063        let author = &mut f.runner.state.config.agents[0];
22064        author.command = vec![
22065            "sh".into(),
22066            "-c".into(),
22067            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22068        ];
22069        author.env.insert(
22070            "PROMPT_FILE".into(),
22071            captured.to_string_lossy().into_owned(),
22072        );
22073        std::fs::write(
22074            &f.reply,
22075            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22076        )
22077        .unwrap();
22078        let m = posted(&mut f).await;
22079        let prompt = std::fs::read_to_string(captured).unwrap();
22080        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22081        assert!(!prompt.contains("in English"));
22082        assert_eq!(m.title, "feat(own): ファイルを追加");
22083        assert!(m.body.contains("own.txt を追加します。"));
22084    }
22085
22086    #[tokio::test]
22087    async fn review_fixes_come_after_the_whole_change_summary() {
22088        let mut f = summary_fixture();
22089        f.runner.state.reviews = vec![
22090            round_with_notes(1, Some("handled the PATH case")),
22091            round_with_notes(2, None),
22092        ];
22093        let m = posted(&mut f).await;
22094        let summary = m.body.find("## Summary").expect("summary");
22095        let fixes = m.body.find("## Review rounds").expect("review rounds");
22096        assert!(summary < fixes, "{}", m.body);
22097        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22098        assert!(m.body.contains("## Final verification"), "{}", m.body);
22099    }
22100
22101    #[tokio::test]
22102    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22103        let mut f = summary_fixture();
22104        let first = posted(&mut f).await;
22105        let first_head = f.runner.state.change_summary.clone().unwrap().head;
22106        // The same head reuses the record: a different reply is not read.
22107        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22108        let again = posted(&mut f).await;
22109        assert_eq!((again.title, again.body), (first.title, first.body));
22110
22111        let g = |args: &[&str]| {
22112            let out = std::process::Command::new("git")
22113                .args(args)
22114                .current_dir(&f.repo)
22115                .quiet()
22116                .output()
22117                .expect("spawn git");
22118            assert!(out.status.success(), "git {args:?}");
22119        };
22120        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22121        g(&["add", "-A"]);
22122        g(&["commit", "-q", "-m", "fix(own): handle more"]);
22123        let second = posted(&mut f).await;
22124        let kept = f.runner.state.change_summary.clone().unwrap();
22125        assert_ne!(kept.head, first_head);
22126        assert!(second.body.contains("Not read yet."), "{}", second.body);
22127        assert!(!second.body.contains("so the first summary is read"));
22128        assert_eq!(second.title, "feat(own): other");
22129    }
22130
22131    #[tokio::test]
22132    async fn an_unavailable_summary_is_stated_and_never_invented() {
22133        let mut f = summary_fixture();
22134        std::fs::write(&f.reply, "no title line here").unwrap();
22135        let m = posted(&mut f).await;
22136        assert!(
22137            m.body
22138                .contains("No description of what this change does could be generated"),
22139            "{}",
22140            m.body
22141        );
22142        assert!(m.body.contains("internal reports remain in local records"));
22143        assert!(f.runner.state.change_summary.is_none());
22144        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22145        // A preview asks nobody and claims nothing.
22146        let winner = f.runner.state.candidates[0].clone();
22147        let preview = f
22148            .runner
22149            .guarded_pr_message(&winner, None, false)
22150            .await
22151            .unwrap()
22152            .unwrap();
22153        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22154    }
22155
22156    #[test]
22157    fn pr_message_review_single_commit_uses_its_subject() {
22158        let state = review_state(&["feat(nats): per-role user"]);
22159        let m = pr_message(&state, 'A');
22160        assert_eq!(m.title, "feat(nats): per-role user");
22161        assert!(!m.body.contains("Review the work already"), "{}", m.body);
22162        assert!(m.body.contains("## Commits under review"), "{}", m.body);
22163    }
22164
22165    #[test]
22166    fn pr_message_review_multi_commit_takes_the_oldest() {
22167        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22168        let m = pr_message(&state, 'A');
22169        assert_eq!(m.title, "feat: the change");
22170        for s in ["feat: the change", "fix: typo", "fix: again"] {
22171            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22172        }
22173    }
22174
22175    #[test]
22176    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22177        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22178            let mut state = review_state(&[first, "fix: later fixup"]);
22179            state.candidates[0].branch = "機能/ログイン".to_owned();
22180            let m = pr_message(&state, 'A');
22181            assert!(
22182                m.title.starts_with("chore: land candidate A of run"),
22183                "{}",
22184                m.title
22185            );
22186        }
22187    }
22188
22189    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22190        BranchFacts {
22191            commits: commits
22192                .iter()
22193                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22194                .collect(),
22195            stat: stat.to_owned(),
22196        }
22197    }
22198
22199    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22200        let mut r = review_round(true, 0, 1, 1, true, true);
22201        r.round = round;
22202        r.fix = notes.map(|n| FixRecord {
22203            agent: "fixer".to_owned(),
22204            addressed: Vec::new(),
22205            rejected: Vec::new(),
22206            notes: n.to_owned(),
22207            committed: true,
22208            failed: None,
22209            duration_ms: 0,
22210            continuation: None,
22211        });
22212        r
22213    }
22214
22215    #[test]
22216    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22217        let state = review_state(&["ignored"]);
22218        let f = facts(
22219            &[
22220                (
22221                    "fix(login): resolve PATH on macOS",
22222                    "Login shells skip rc files.",
22223                ),
22224                ("fix: address review", ""),
22225            ],
22226            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22227        );
22228        let m = pr_message_with(&state, 'A', Some(&f));
22229        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22230        assert!(
22231            m.body
22232                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22233            "{}",
22234            m.body
22235        );
22236        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22237        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22238        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22239        for banned in [
22240            "Review the work already",
22241            "no task statement",
22242            "Original task",
22243        ] {
22244            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22245        }
22246        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22247    }
22248
22249    #[test]
22250    fn pr_message_review_truncates_a_huge_first_commit_body() {
22251        let state = review_state(&["ignored"]);
22252        let f = facts(
22253            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22254            "s",
22255        );
22256        let m = pr_message_with(&state, 'A', Some(&f));
22257        assert!(m.body.len() < 30_000, "{}", m.body.len());
22258        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22259        assert!(
22260            m.body.contains("- ... 1 more commit(s)"),
22261            "{}",
22262            m.body.len()
22263        );
22264        assert!(m.body.ends_with("magi:candidate-a\n"));
22265    }
22266
22267    #[test]
22268    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22269        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22270        assert_eq!(m.title, "feat: x");
22271        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22272        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22273        assert!(!m.body.contains("no task statement"), "{}", m.body);
22274    }
22275
22276    #[test]
22277    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22278        let mut state = review_state(&["日本語の件名"]);
22279        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22280        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22281        state.candidates[0].branch = "機能/ログイン".to_owned();
22282        assert!(
22283            pr_message(&state, 'A')
22284                .title
22285                .starts_with("chore: land candidate A")
22286        );
22287    }
22288
22289    #[test]
22290    fn pr_message_review_fixes_survive_a_clean_final_round() {
22291        let mut state = review_state(&["feat: x"]);
22292        state.reviews = vec![
22293            round_with_notes(1, Some("handled the PATH case")),
22294            round_with_notes(2, None),
22295        ];
22296        let body = pr_message(&state, 'A').body;
22297        assert!(
22298            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22299            "{body}"
22300        );
22301        assert!(body.contains("### Round 1"), "{body}");
22302
22303        state.reviews = vec![
22304            round_with_notes(1, Some("first fix")),
22305            round_with_notes(2, Some("")),
22306            round_with_notes(3, Some("second fix")),
22307            round_with_notes(4, None),
22308        ];
22309        let body = pr_message(&state, 'A').body;
22310        assert!(
22311            body.contains("### Round 1") && !body.contains("first fix"),
22312            "{body}"
22313        );
22314        assert!(
22315            body.contains("### Round 3") && !body.contains("second fix"),
22316            "{body}"
22317        );
22318        assert!(body.contains("### Round 2"), "{body}");
22319    }
22320
22321    const FINAL: &str = "abcdef1234567890";
22322
22323    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22324        let mut r = round_with_notes(round, None);
22325        r.verified_head = verified.map(str::to_owned);
22326        r
22327    }
22328
22329    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22330        state.gate = vec![CommandOutcome {
22331            command: "gate".to_owned(),
22332            code: Some(0),
22333            output_tail: String::new(),
22334            duration_ms: 0,
22335            resource_blocked: false,
22336        }];
22337        state.gate_ran = true;
22338        state.gate_head = head.map(str::to_owned);
22339    }
22340
22341    fn body_at(state: &RunState) -> String {
22342        pr_message_raw(
22343            state,
22344            'A',
22345            None,
22346            crate::pr_outcome::FinalHead::At(FINAL),
22347            ChangeNote::NotAsked,
22348        )
22349        .body
22350    }
22351
22352    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22353
22354    #[test]
22355    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22356        let mut state = review_state(&["feat: x"]);
22357        state.reviews = vec![
22358            round_with_notes(1, Some(SANDBOXED)),
22359            passing_round(2, Some(FINAL)),
22360        ];
22361        ok_gate(&mut state, Some(FINAL));
22362        let body = body_at(&state);
22363        assert!(!body.contains("Handled the PATH case."), "{body}");
22364        assert!(!body.contains("could not commit"), "{body}");
22365        assert!(!body.contains("could not run"), "{body}");
22366        assert!(
22367            body.contains("The fix is committed on the branch"),
22368            "{body}"
22369        );
22370        assert!(!body.contains("sandbox"), "{body}");
22371        assert!(
22372            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22373            "{body}"
22374        );
22375        assert!(
22376            body.contains("Gate (`verify.gate`): passed on the final commit"),
22377            "{body}"
22378        );
22379        // The raw report is untouched in the record.
22380        assert!(
22381            state.reviews[0]
22382                .fix
22383                .as_ref()
22384                .unwrap()
22385                .notes
22386                .contains("could not commit")
22387        );
22388    }
22389
22390    #[test]
22391    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22392        let mut state = review_state(&["feat: x"]);
22393        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22394        r1.fix.as_mut().unwrap().committed = false;
22395        let mut r2 = passing_round(2, Some(FINAL));
22396        r2.e2e[0].code = Some(1);
22397        state.reviews = vec![r1, r2];
22398        let body = body_at(&state);
22399        assert!(!body.contains("could not commit"), "{body}");
22400        assert!(!body.contains("could not run"), "{body}");
22401        assert!(body.contains("This round produced no commit"), "{body}");
22402        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22403        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22404        assert!(!body.contains("passed"), "{body}");
22405    }
22406
22407    #[test]
22408    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22409        let mut state = review_state(&["feat: x"]);
22410        state.reviews = vec![
22411            round_with_notes(1, Some(SANDBOXED)),
22412            passing_round(2, Some("0123456789abcdef")),
22413        ];
22414        ok_gate(&mut state, Some("0123456789abcdef"));
22415        let body = body_at(&state);
22416        assert!(!body.contains("HTTP tests could not run"), "{body}");
22417        // The commit claim is still settled by the structured record.
22418        assert!(!body.contains("could not commit"), "{body}");
22419        assert!(
22420            body.contains("passed on an earlier commit (`0123456`)"),
22421            "{body}"
22422        );
22423        assert!(!body.contains("passed on the final commit"), "{body}");
22424
22425        // A gate recorded before `gate_head` existed is "not recorded".
22426        ok_gate(&mut state, None);
22427        let body = body_at(&state);
22428        assert!(
22429            body.contains("passed on a commit that was not recorded"),
22430            "{body}"
22431        );
22432    }
22433
22434    #[test]
22435    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22436        let state = state_with_summary(
22437            "add retries to the client",
22438            "TITLE: feat: retries\n\nDid it.",
22439        );
22440        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22441        assert_eq!(m.title, "feat: retries");
22442        assert!(
22443            m.body.contains("<summary>Original task</summary>"),
22444            "{}",
22445            m.body
22446        );
22447        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22448        assert!(
22449            m.body
22450                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22451            "{}",
22452            m.body
22453        );
22454    }
22455
22456    #[test]
22457    fn pr_message_review_bounds_a_long_english_subject() {
22458        let long = format!("feat: {}", "word ".repeat(100));
22459        let m = pr_message(&review_state(&[&long]), 'A');
22460        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22461        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22462    }
22463
22464    #[test]
22465    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22466        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22467        let m = pr_message(&state, 'A');
22468        assert_eq!(m.title, "add retries");
22469        assert!(m.body.contains("<summary>Original task</summary>"));
22470        assert!(!m.body.contains("Commits under review"));
22471        assert_eq!(landing_subject_source(&state), state.instruction);
22472    }
22473
22474    #[test]
22475    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22476        let state = review_state(&["feat: the change", "fix: typo"]);
22477        let source = landing_subject_source(&state);
22478        assert_eq!(land::merge_subject("", &source), "feat: the change");
22479        assert_eq!(
22480            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22481            "feat: the change"
22482        );
22483        // An operator's rename still wins.
22484        assert_eq!(
22485            land::merge_subject("feat: renamed by hand", &source),
22486            "feat: renamed by hand"
22487        );
22488        let blank = review_state(&["日本語"]);
22489        assert!(
22490            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22491        );
22492    }
22493
22494    #[test]
22495    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22496        let state = review_state(&["feat: the change"]);
22497        let source = landing_subject_source(&state);
22498        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22499        assert_eq!(
22500            land::merge_subject(landing_title(&state, old), &source),
22501            "feat: the change"
22502        );
22503        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22504        let task = state_with_summary("add retries", "");
22505        assert_eq!(landing_title(&task, old), old);
22506    }
22507
22508    #[test]
22509    fn pr_message_describes_the_change_not_the_task() {
22510        let state = state_with_summary(
22511            "今回やってほしいこと: results projector を直す",
22512            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22513        );
22514        let m = pr_message(&state, 'A');
22515        assert_eq!(m.title, "fix(web): batch the runs list reads");
22516        assert!(
22517            m.body.starts_with("## Summary\n\n- reads run.json once"),
22518            "{}",
22519            m.body
22520        );
22521        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22522        let task_at = m.body.find("今回やってほしいこと").unwrap();
22523        let details_at = m.body.find("<details>").unwrap();
22524        assert!(
22525            details_at < task_at,
22526            "the task lives inside <details>: {}",
22527            m.body
22528        );
22529        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22530        assert!(m.body.contains("magi:candidate-a"));
22531    }
22532
22533    #[test]
22534    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22535        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22536        let m = pr_message(&state, 'A');
22537        assert_eq!(m.title, "add retries");
22538        assert!(
22539            m.body.contains("## Summary\n\n- did some things"),
22540            "{}",
22541            m.body
22542        );
22543
22544        let none = RunState::new(
22545            PathBuf::from("/repo"),
22546            "main".to_owned(),
22547            "abc1234".to_owned(),
22548            "add retries".to_owned(),
22549            Config::default(),
22550        );
22551        let m = pr_message(&none, 'A');
22552        assert_eq!(m.title, "add retries");
22553        assert!(!m.body.contains("## Summary"), "{}", m.body);
22554    }
22555
22556    #[test]
22557    fn pr_message_refuses_the_candidate_commit_subject() {
22558        for bad in [
22559            "TITLE: magi: candidate A (uncommitted work)",
22560            "TITLE: chore: stuff (uncommitted work)",
22561            "TITLE:   ",
22562        ] {
22563            let state = state_with_summary("add retries", bad);
22564            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22565        }
22566    }
22567
22568    #[test]
22569    fn pr_message_bounds_a_very_long_task_and_title() {
22570        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22571        let state = state_with_summary(&long, "- nothing");
22572        let m = pr_message(&state, 'A');
22573        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22574        assert!(!m.title.contains('\n'));
22575
22576        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22577        let m = pr_message(&state, 'A');
22578        assert!(m.title.starts_with("feat: "));
22579        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22580        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22581    }
22582
22583    fn long_title_of(instruction: &str) -> String {
22584        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22585    }
22586
22587    #[test]
22588    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22589        let first = "Make the landing path keep a readable title for long tasks";
22590        let line = format!(
22591            "{first}. {}",
22592            "And then keep going with more words ".repeat(20)
22593        );
22594        let t = long_title_of(&line);
22595        assert_eq!(t, first);
22596        assert!(!t.starts_with("chore: land"));
22597    }
22598
22599    #[test]
22600    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22601        let line = "word ".repeat(200);
22602        let t = long_title_of(&line);
22603        assert!(t.ends_with("word..."), "{t}");
22604        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22605    }
22606
22607    #[test]
22608    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22609        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22610            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22611        }
22612    }
22613
22614    #[test]
22615    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22616        let mut state = state_with_summary("再試行を追加する", "- nothing");
22617        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22618        state.config.graph.github_language = "ja".into();
22619        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22620    }
22621
22622    #[test]
22623    fn pr_message_title_limit_is_exact() {
22624        let at = "a".repeat(PR_TITLE_MAX);
22625        assert_eq!(long_title_of(&at), at);
22626        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22627        assert!(over.ends_with("..."), "{over}");
22628        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22629    }
22630
22631    #[test]
22632    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22633        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22634        let t = long_title_of(&line);
22635        assert!(t.ends_with("..."), "{t}");
22636        assert!(t.is_ascii(), "{t}");
22637    }
22638
22639    #[test]
22640    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22641        let line = format!(
22642            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22643            "plus more filler words ".repeat(20)
22644        );
22645        let t = long_title_of(&line);
22646        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22647    }
22648
22649    #[test]
22650    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22651        let line = format!("\n\n# {}", "title words ".repeat(40));
22652        let state = state_with_summary(&line, "- nothing");
22653        let m = pr_message(&state, 'A');
22654        assert!(m.title.starts_with("title words"), "{}", m.title);
22655        assert_eq!(
22656            land::merge_subject(&m.title, &landing_subject_source(&state)),
22657            m.title
22658        );
22659        // An operator's rename wins untouched.
22660        assert_eq!(
22661            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22662            "feat: renamed by hand"
22663        );
22664    }
22665
22666    #[tokio::test]
22667    async fn github_text_rewrite_is_bounded_and_falls_back() {
22668        crate::run::pin_test_home();
22669        for (reply, accepted) in [
22670            (
22671                "TITLE: fix: retries\nAdd retries for failed requests.",
22672                true,
22673            ),
22674            (
22675                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22676                false,
22677            ),
22678            (
22679                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22680                true,
22681            ),
22682            (
22683                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22684                true,
22685            ),
22686            (
22687                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22688                true,
22689            ),
22690            (
22691                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22692                false,
22693            ),
22694            ("TITLE: fix: retries\n", false),
22695        ] {
22696            let dir = tempfile::tempdir().unwrap();
22697            let mut runner = runner_at(RunStatus::Gating);
22698            runner.state = state_with_summary(
22699                "add retries",
22700                "TITLE: fix: retries\n日本語の説明を書きます。",
22701            );
22702            runner.state.repo = dir.path().to_owned();
22703            runner.state.candidates[0].worktree = dir.path().to_owned();
22704            let mut author = spec("alpha");
22705            author.command = vec![
22706                "sh".into(),
22707                "-c".into(),
22708                "printf '%s' \"$REWRITE_REPLY\"".into(),
22709            ];
22710            author.env.insert("REWRITE_REPLY".into(), reply.into());
22711            runner.state.config.agents = vec![author];
22712            seed_public_summary(&mut runner);
22713            let winner = runner.state.candidates[0].clone();
22714            let first = runner
22715                .guarded_pr_message(&winner, None, true)
22716                .await
22717                .unwrap();
22718            let message = if accepted {
22719                first.expect("a passing rewrite never asks")
22720            } else {
22721                // Withheld: the run parks on a question and posts nothing yet.
22722                assert!(first.is_none());
22723                assert!(runner.state.parked);
22724                let store = ask::Questions::open();
22725                let asked: Vec<_> = store
22726                    .list()
22727                    .into_iter()
22728                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22729                    .collect();
22730                assert_eq!(asked.len(), 1);
22731                assert!(!asked[0].detail.contains("secret"));
22732                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22733                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22734                assert_eq!(
22735                    asked[0]
22736                        .choices
22737                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22738                    rewrite.is_some()
22739                );
22740                // Silence: the question lapses and the neutral text is used,
22741                // without a second rewrite call or a second question.
22742                store
22743                    .update(&asked[0].id, |q| {
22744                        q.abandon("test");
22745                        Ok(())
22746                    })
22747                    .unwrap();
22748                let again = runner
22749                    .guarded_pr_message(&winner, None, true)
22750                    .await
22751                    .unwrap()
22752                    .expect("an abandoned question falls back");
22753                let n = store
22754                    .list()
22755                    .into_iter()
22756                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22757                    .count();
22758                assert_eq!(n, 1, "asked once per rejected text");
22759                again
22760            };
22761            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22762            assert!(!message.body.contains("secret"));
22763            assert!(!message.body.contains("prefix"));
22764            assert!(!message.title.contains("secret"));
22765            assert_eq!(
22766                message.body.contains("Add retries for failed requests."),
22767                accepted
22768            );
22769            assert_eq!(
22770                runner.state.seats["github-text-rewrite"].turns, 1,
22771                "only one rewrite invocation"
22772            );
22773            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22774            if accepted {
22775                assert!(runner.state.github_text.is_none());
22776                assert!(!runner.state.parked);
22777                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22778                assert!(message.body.contains("## Final verification"));
22779                assert!(message.body.contains("Evidence: final diff"));
22780                assert!(
22781                    message
22782                        .body
22783                        .contains(&format!("magi:run/{}", runner.state.id))
22784                );
22785            }
22786            if !accepted {
22787                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22788                assert!(
22789                    message
22790                        .body
22791                        .contains(&format!("magi:run/{}", runner.state.id))
22792                );
22793            }
22794        }
22795    }
22796
22797    fn scripted(id: &str, reply: &str) -> AgentSpec {
22798        let mut a = spec(id);
22799        a.command = vec![
22800            "sh".into(),
22801            "-c".into(),
22802            "printf '%s' \"$REWRITE_REPLY\"".into(),
22803        ];
22804        a.env.insert("REWRITE_REPLY".into(), reply.into());
22805        a
22806    }
22807
22808    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22809    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22810
22811    /// Gate tests inject generated public output, not internal reports.
22812    fn seed_public_summary(runner: &mut Runner) {
22813        let repo = &runner.state.repo;
22814        let g = |args: &[&str]| {
22815            let out = std::process::Command::new("git")
22816                .args(args)
22817                .current_dir(repo)
22818                .quiet()
22819                .output()
22820                .unwrap();
22821            assert!(out.status.success(), "git {args:?}");
22822            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22823        };
22824        g(&["init", "-q", "-b", "main"]);
22825        g(&[
22826            "-c",
22827            "user.name=Test",
22828            "-c",
22829            "user.email=test@example.invalid",
22830            "commit",
22831            "-q",
22832            "--allow-empty",
22833            "-m",
22834            "feat: fixture",
22835        ]);
22836        let head = g(&["rev-parse", "HEAD"]);
22837        g(&["update-ref", "refs/remotes/origin/main", &head]);
22838        let evidence = crate::pr_outcome::summary_evidence(
22839            &runner.state,
22840            crate::pr_outcome::FinalHead::At(&head),
22841        );
22842        let generated = &runner.state.candidates[0].summary;
22843        runner.state.change_summary = Some(crate::run::ChangeSummary {
22844            base: head.clone(),
22845            head,
22846            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22847            title: summary_title(generated).unwrap(),
22848            body: summary_without_title(generated),
22849        });
22850    }
22851
22852    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22853        let mut runner = runner_at(RunStatus::Gating);
22854        runner.state = state_with_summary("add retries", JA_SUMMARY);
22855        if review_only {
22856            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22857        }
22858        runner.state.repo = dir.to_owned();
22859        runner.state.candidates[0].worktree = dir.to_owned();
22860        runner.state.config.agents = agents;
22861        seed_public_summary(&mut runner);
22862        runner
22863    }
22864
22865    fn gate_questions(runner: &Runner) -> usize {
22866        ask::Questions::open()
22867            .list()
22868            .into_iter()
22869            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22870            .count()
22871    }
22872
22873    #[tokio::test]
22874    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22875        crate::run::pin_test_home();
22876        let dir = tempfile::tempdir().unwrap();
22877        let mut runner = rewrite_runner(
22878            dir.path(),
22879            true,
22880            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22881        );
22882        runner.state.config.roles.language_judge =
22883            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22884        let winner = runner.state.candidates[0].clone();
22885        let msg = runner
22886            .guarded_pr_message(&winner, None, true)
22887            .await
22888            .unwrap()
22889            .expect("an English rewrite posts without asking");
22890        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22891        assert!(msg.body.contains("<details>kept</details>"));
22892        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22893        assert_eq!(gate_questions(&runner), 0);
22894        assert!(runner.state.github_text.is_none());
22895        assert!(!runner.state.seats.contains_key("impl-A"));
22896        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22897    }
22898
22899    #[tokio::test]
22900    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22901        crate::run::pin_test_home();
22902        let dir = tempfile::tempdir().unwrap();
22903        let mut runner = rewrite_runner(
22904            dir.path(),
22905            false,
22906            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22907        );
22908        runner.state.config.roles.language_judge =
22909            Some(crate::config::AgentChoice::One("beta".into()));
22910        runner.state.seats.insert(
22911            "impl-A".into(),
22912            SeatState::new("impl-A", "alpha", runner.state.seed),
22913        );
22914        let winner = runner.state.candidates[0].clone();
22915        let msg = runner
22916            .guarded_pr_message(&winner, None, true)
22917            .await
22918            .unwrap()
22919            .expect("rewritten");
22920        assert!(msg.body.contains("<details>kept</details>"));
22921        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22922        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22923        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22924    }
22925
22926    #[tokio::test]
22927    async fn rewrite_reuses_a_resumable_implementer_session() {
22928        crate::run::pin_test_home();
22929        let dir = tempfile::tempdir().unwrap();
22930        let mut runner = rewrite_runner(
22931            dir.path(),
22932            false,
22933            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22934        );
22935        runner.state.config.roles.language_judge =
22936            Some(crate::config::AgentChoice::One("beta".into()));
22937        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22938        seat.turns = 1;
22939        runner.state.seats.insert("impl-A".into(), seat);
22940        let winner = runner.state.candidates[0].clone();
22941        runner
22942            .guarded_pr_message(&winner, None, true)
22943            .await
22944            .unwrap()
22945            .expect("rewritten");
22946        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22947        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22948    }
22949
22950    #[tokio::test]
22951    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22952        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22953        // session, was out of quota, and nothing else was asked.
22954        crate::run::pin_test_home();
22955        let dir = tempfile::tempdir().unwrap();
22956        let mut exhausted = spec("alpha");
22957        exhausted.command = vec!["false".into()];
22958        let mut runner = rewrite_runner(
22959            dir.path(),
22960            false,
22961            vec![exhausted, scripted("beta", EN_REWRITE)],
22962        );
22963        // The implementer's own agent is in the role too: it is not asked twice.
22964        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22965            "alpha".into(),
22966            "beta".into(),
22967        ]));
22968        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22969        seat.turns = 1;
22970        runner.state.seats.insert("impl-A".into(), seat);
22971        let winner = runner.state.candidates[0].clone();
22972        let msg = runner
22973            .guarded_pr_message(&winner, None, true)
22974            .await
22975            .unwrap()
22976            .expect("the role rewrites after the implementer fails");
22977        assert!(msg.body.contains("`src/net.rs`"));
22978        assert_eq!(
22979            runner.state.seats["impl-A"].turns, 2,
22980            "the implementer was tried once"
22981        );
22982        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22983        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22984    }
22985
22986    #[tokio::test]
22987    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22988        crate::run::pin_test_home();
22989        let dir = tempfile::tempdir().unwrap();
22990        let mut broken = spec("bad");
22991        broken.command = vec!["false".into()];
22992        let mut runner = rewrite_runner(
22993            dir.path(),
22994            true,
22995            vec![
22996                broken,
22997                scripted("noformat", "Add retries."),
22998                scripted("good", EN_REWRITE),
22999            ],
23000        );
23001        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23002            "bad".into(),
23003            "noformat".into(),
23004            "good".into(),
23005        ]));
23006        let winner = runner.state.candidates[0].clone();
23007        let msg = runner
23008            .guarded_pr_message(&winner, None, true)
23009            .await
23010            .unwrap()
23011            .expect("the third agent rewrites");
23012        assert!(msg.body.contains("`src/net.rs`"));
23013        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23014        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23015    }
23016
23017    #[tokio::test]
23018    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23019        crate::run::pin_test_home();
23020        // (agents, role): still Japanese; no TITLE; nothing resolvable.
23021        for (agents, role, saved) in [
23022            (
23023                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23024                Some("beta"),
23025                true,
23026            ),
23027            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23028            (Vec::new(), None, false),
23029        ] {
23030            let dir = tempfile::tempdir().unwrap();
23031            let mut runner = rewrite_runner(dir.path(), true, agents);
23032            runner.state.config.roles.language_judge =
23033                role.map(|r| crate::config::AgentChoice::One(r.into()));
23034            let winner = runner.state.candidates[0].clone();
23035            let first = runner
23036                .guarded_pr_message(&winner, None, true)
23037                .await
23038                .unwrap();
23039            assert!(first.is_none(), "withheld text parks the run");
23040            assert!(runner.state.parked);
23041            assert_eq!(gate_questions(&runner), 1);
23042            let g = runner.state.github_text.as_ref().unwrap();
23043            assert_eq!(g.rewrite.is_some(), saved);
23044            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23045        }
23046    }
23047
23048    #[tokio::test]
23049    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23050        crate::run::pin_test_home();
23051        let dir = tempfile::tempdir().unwrap();
23052        let mut runner = runner_at(RunStatus::Gating);
23053        runner.state =
23054            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23055        runner.state.repo = dir.path().to_owned();
23056        runner.state.candidates[0].worktree = dir.path().to_owned();
23057        let mut author = spec("alpha");
23058        author.command = vec![
23059            "sh".into(),
23060            "-c".into(),
23061            "printf '%s' \"$REWRITE_REPLY\"".into(),
23062        ];
23063        author.env.insert(
23064            "REWRITE_REPLY".into(),
23065            format!(
23066                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23067                "English sentence. ".repeat(600)
23068            ),
23069        );
23070        runner.state.config.agents = vec![author];
23071        seed_public_summary(&mut runner);
23072        let winner = runner.state.candidates[0].clone();
23073        assert!(
23074            runner
23075                .guarded_pr_message(&winner, None, true)
23076                .await
23077                .unwrap()
23078                .is_none()
23079        );
23080        let record = runner.state.github_text.clone().unwrap();
23081        let rewrite = record.rewrite.clone().unwrap();
23082        assert_eq!(rewrite.failed, ["body-language"]);
23083        assert!(
23084            rewrite
23085                .body
23086                .ends_with(&format!("magi:run/{}", runner.state.id))
23087        );
23088        let store = ask::Questions::open();
23089        let q = store.get(record.question.as_ref().unwrap()).unwrap();
23090        assert!(
23091            q.choices
23092                .contains(&crate::github_text::USE_REWRITE.to_owned())
23093        );
23094        for label in [
23095            "ORIGINAL",
23096            "FALLBACK",
23097            "REWRITE",
23098            "Rules still failed: body-language",
23099            "truncated",
23100        ] {
23101            assert!(q.detail.contains(label), "{label}");
23102        }
23103        let artifact = std::fs::read_to_string(crate::run::artifact_path(
23104            &runner.state,
23105            &crate::github_text::artifact_name(&record.fingerprint),
23106        ))
23107        .unwrap();
23108        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23109        // Restart from serialized state, then retry after a bad owner title.
23110        runner.state =
23111            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23112        store
23113            .update(&q.id, |q| {
23114                q.say("まだ日本語")?;
23115                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23116            })
23117            .unwrap();
23118        assert!(
23119            runner
23120                .guarded_pr_message(&winner, None, true)
23121                .await
23122                .unwrap()
23123                .is_none()
23124        );
23125        let retry = runner.state.github_text.as_ref().unwrap();
23126        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23127        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23128        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23129        assert!(
23130            q.posting_comparison
23131                .as_ref()
23132                .unwrap()
23133                .matches_detail(&q.detail)
23134        );
23135        store
23136            .update(&q.id, |q| {
23137                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23138            })
23139            .unwrap();
23140        let message = runner
23141            .guarded_pr_message(&winner, None, true)
23142            .await
23143            .unwrap()
23144            .unwrap();
23145        assert_eq!(message.title, rewrite.title);
23146        assert_eq!(message.body, rewrite.body);
23147        assert_eq!(
23148            runner.state.github_text.as_ref().unwrap().choice,
23149            Some(GithubTextChoice::Rewrite)
23150        );
23151        let again = runner
23152            .guarded_pr_message(&winner, None, true)
23153            .await
23154            .unwrap()
23155            .unwrap();
23156        assert_eq!(again.title, message.title);
23157        assert_eq!(again.body, message.body);
23158        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23159        assert!(
23160            !runner
23161                .state
23162                .events
23163                .iter()
23164                .any(|e| e.message.contains(&rewrite.title)
23165                    || e.message.contains("English sentence."))
23166        );
23167    }
23168
23169    #[tokio::test]
23170    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23171        crate::run::pin_test_home();
23172        let dir = tempfile::tempdir().unwrap();
23173        let mut runner = runner_at(RunStatus::Gating);
23174        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23175        runner.state.repo = dir.path().to_owned();
23176        runner.state.candidates[0].worktree = dir.path().to_owned();
23177        seed_public_summary(&mut runner);
23178        let winner = runner.state.candidates[0].clone();
23179        assert!(
23180            runner
23181                .guarded_pr_message(&winner, None, true)
23182                .await
23183                .unwrap()
23184                .is_none()
23185        );
23186        let record = runner.state.github_text.as_ref().unwrap();
23187        assert!(record.rewrite.is_none());
23188        // Records written before the additive field deserialize with no rewrite.
23189        let mut old = serde_json::to_value(record).unwrap();
23190        old.as_object_mut().unwrap().remove("rewrite");
23191        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23192        assert!(old.rewrite.is_none());
23193        let id = record.question.clone().unwrap();
23194        ask::Questions::open()
23195            .update(&id, |q| {
23196                q.choices.push(crate::github_text::USE_REWRITE.into());
23197                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23198            })
23199            .unwrap();
23200        let message = runner
23201            .guarded_pr_message(&winner, None, true)
23202            .await
23203            .unwrap()
23204            .unwrap();
23205        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23206        assert_eq!(
23207            runner.state.github_text.as_ref().unwrap().choice,
23208            Some(GithubTextChoice::Fallback)
23209        );
23210    }
23211
23212    #[tokio::test]
23213    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23214        crate::run::pin_test_home();
23215        let dir = tempfile::tempdir().unwrap();
23216        let mut runner = runner_at(RunStatus::Gating);
23217        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23218        runner.state.repo = dir.path().to_owned();
23219        runner.state.candidates[0].worktree = dir.path().to_owned();
23220        seed_public_summary(&mut runner);
23221        let winner = runner.state.candidates[0].clone();
23222        assert!(
23223            runner
23224                .guarded_pr_message(&winner, None, true)
23225                .await
23226                .unwrap()
23227                .is_none()
23228        );
23229        let store = ask::Questions::open();
23230        let find = |run: &str| {
23231            store
23232                .list()
23233                .into_iter()
23234                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23235                .unwrap()
23236        };
23237        let run = runner.state.id.clone();
23238        let q = find(&run);
23239        let fp = runner
23240            .state
23241            .github_text
23242            .as_ref()
23243            .unwrap()
23244            .fingerprint
23245            .clone();
23246        let name = crate::github_text::artifact_name(&fp);
23247        let path = crate::run::artifact_path(&runner.state, &name);
23248        assert!(q.detail.contains(&path.display().to_string()));
23249        assert!(q.detail.contains("title-language: title"));
23250        let saved = std::fs::read_to_string(&path).expect("artifact written");
23251        assert!(saved.contains("日本語のタイトル"));
23252        assert!(q.cwd.is_none());
23253        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23254        // A bad replacement is refused and asked about once more.
23255        let say = |id: &str, text: &str, choice: &str| {
23256            store
23257                .update(id, |q| {
23258                    q.thread.push(ask::Turn {
23259                        who: ask::Who::Operator,
23260                        body: text.to_owned(),
23261                        at: jiff::Timestamp::now(),
23262                        note: None,
23263                    });
23264                    q.answer(ask::Answer::Choice(choice.to_owned()))
23265                })
23266                .unwrap();
23267        };
23268        say(&q.id, "まだ日本語", "use my text");
23269        assert!(
23270            runner
23271                .guarded_pr_message(&winner, None, true)
23272                .await
23273                .unwrap()
23274                .is_none()
23275        );
23276        let second = find(&run);
23277        assert_ne!(second.id, q.id);
23278        assert!(second.detail.contains("did not pass"));
23279        // The retry still compares the original text; the rejected
23280        // replacement is named by category only.
23281        assert!(second.detail.contains("日本語のタイトル"));
23282        assert!(!second.detail.contains("まだ日本語"));
23283        assert!(second.detail.contains(&path.display().to_string()));
23284        say(&second.id, "fix: retry failed requests", "use my text");
23285        let message = runner
23286            .guarded_pr_message(&winner, None, true)
23287            .await
23288            .unwrap()
23289            .expect("a vetted replacement is posted");
23290        assert_eq!(message.title, "fix: retry failed requests");
23291        assert_eq!(
23292            runner
23293                .state
23294                .github_text
23295                .as_ref()
23296                .unwrap()
23297                .chosen_title
23298                .as_deref(),
23299            Some("fix: retry failed requests")
23300        );
23301        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23302        // Resumed again: the saved decision is applied, nothing is re-asked.
23303        let again = runner
23304            .guarded_pr_message(&winner, None, true)
23305            .await
23306            .unwrap()
23307            .unwrap();
23308        assert_eq!(again.title, message.title);
23309        assert_eq!(
23310            store
23311                .list()
23312                .into_iter()
23313                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23314                .count(),
23315            2
23316        );
23317    }
23318
23319    #[test]
23320    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23321        // What magi itself writes stays English under any configured language,
23322        // so a future localisation of these headings fails here. (The agents'
23323        // own text is also checked by the posting gate.)
23324        let mut state = state_with_summary(
23325            "add retries",
23326            "TITLE: fix(web): batch reads\n- reads run.json once",
23327        );
23328        state.config.graph.language = "ja".to_owned();
23329        let m = pr_message(&state, 'A');
23330        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23331        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23332
23333        // The task is the operator's own text: it goes in untouched, and the
23334        // fallback title (no summary) may be in its language too.
23335        let task = "今回やってほしいこと: results projector を直す";
23336        let mut state = state_with_summary(task, "- no title line");
23337        state.config.graph.language = "ja".to_owned();
23338        let m = pr_message(&state, 'A');
23339        assert_eq!(
23340            m.title,
23341            format!("chore: land candidate A of run {}", state.id)
23342        );
23343        assert!(
23344            m.body.contains(&format!(
23345                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23346            )),
23347            "{}",
23348            m.body
23349        );
23350    }
23351
23352    #[test]
23353    fn pr_message_scrubs_home_paths_and_addresses() {
23354        let state = state_with_summary(
23355            "fix it in /Users/someone/src/x",
23356            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23357        );
23358        let m = pr_message(&state, 'A');
23359        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23360            assert!(!m.body.contains(leak), "{}", m.body);
23361        }
23362        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23363    }
23364
23365    #[test]
23366    fn pr_message_survives_a_task_that_closes_details() {
23367        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23368        let m = pr_message(&state, 'A');
23369        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23370    }
23371
23372    #[test]
23373    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23374        let cmd = manual_merge_command(
23375            MergeStyle::Squash,
23376            Path::new("/repo"),
23377            "b",
23378            "fix: \"quoted\" $(x) `y`\n\nbody",
23379        );
23380        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23381    }
23382
23383    #[test]
23384    fn manual_merge_command_matches_the_configured_style() {
23385        let repo = Path::new("/repo");
23386        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23387
23388        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23389        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23390
23391        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23392        assert_eq!(
23393            squash,
23394            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23395             \"Merge magi run 0832 (candidate A)\""
23396        );
23397
23398        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23399        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23400    }
23401
23402    #[test]
23403    fn a_nudge_gets_a_quarter_of_the_budget() {
23404        // The judge and implement budgets magi ships with.
23405        assert_eq!(retry_budget(secs(1200), true), secs(300));
23406        assert_eq!(retry_budget(secs(3600), true), secs(900));
23407    }
23408
23409    #[test]
23410    fn a_resent_prompt_keeps_the_whole_budget() {
23411        // The seat kept no context, so the retry is the original job again and
23412        // shortening it would only guarantee a second failure.
23413        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23414        assert_eq!(retry_budget(secs(60), false), secs(60));
23415    }
23416
23417    #[test]
23418    fn the_floor_never_exceeds_the_original_budget() {
23419        // A short configured timeout must not be *raised* by the floor: the
23420        // operator asked for a bound, and a retry may not outlast the attempt
23421        // it is retrying.
23422        assert_eq!(retry_budget(secs(60), true), secs(60));
23423        assert_eq!(retry_budget(secs(480), true), secs(120));
23424        assert_eq!(retry_budget(secs(0), true), secs(0));
23425    }
23426
23427    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23428        agent::CommandEvidence {
23429            id: "item1".to_owned(),
23430            description: "cargo test".to_owned(),
23431            exit_code,
23432            result_summary: String::new(),
23433            source: "codex".to_owned(),
23434        }
23435    }
23436
23437    #[test]
23438    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23439        // No evidence is not the same fact as unconfirmed evidence: a
23440        // backend with no adapter, or a reply that ran no commands at all,
23441        // must not be misread as carrying a dangling job.
23442        assert!(!has_unconfirmed_command(&[]));
23443    }
23444
23445    #[test]
23446    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23447        // Deliberately not a check on the exit code's *value*: a fixer
23448        // legitimately runs something that fails mid-iteration before it
23449        // succeeds, and that must never by itself reopen a valid report.
23450        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23451        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23452        assert!(!has_unconfirmed_command(&[
23453            evidence(Some(0)),
23454            evidence(Some(101))
23455        ]));
23456    }
23457
23458    #[test]
23459    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23460        assert!(has_unconfirmed_command(&[
23461            evidence(Some(0)),
23462            evidence(None)
23463        ]));
23464    }
23465
23466    #[test]
23467    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23468        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23469        assert_eq!(
23470            verified_noop_claim(true, &[], text).as_deref(),
23471            Some("already fixed by b32cfc4, on main.")
23472        );
23473    }
23474
23475    #[test]
23476    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23477        // A timeout or a bad exit code reads as the ordinary loss it is,
23478        // whatever the reply's own prose claims.
23479        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23480        assert!(verified_noop_claim(false, &[], text).is_none());
23481    }
23482
23483    #[test]
23484    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23485        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23486        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23487        // A confirmed command alongside the marker is fine.
23488        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23489    }
23490
23491    #[test]
23492    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23493        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23494    }
23495
23496    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23497    /// pair, in order, labelled A, B, C, ...
23498    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23499        runner.state.candidates = shape
23500            .iter()
23501            .enumerate()
23502            .map(|(i, &(empty, verified))| Candidate {
23503                index: i,
23504                label: (b'A' + i as u8) as char,
23505                agent: "sonnet".to_owned(),
23506                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23507                worktree: PathBuf::from(format!("/wt/{i}")),
23508                summary: String::new(),
23509                stat: String::new(),
23510                files: 0,
23511                commits: 0,
23512                empty,
23513                failed: None,
23514                verified_noop: verified.map(str::to_owned),
23515                duration_ms: 0,
23516                folded: false,
23517            })
23518            .collect();
23519    }
23520
23521    #[test]
23522    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23523        ask_test_home();
23524        let mut runner = runner_at(RunStatus::Implementing);
23525        set_candidates(
23526            &mut runner,
23527            &[
23528                (true, Some("already on main at b32cfc4")),
23529                (true, Some("same fix, see the existing test")),
23530            ],
23531        );
23532
23533        runner
23534            .after_implement()
23535            .expect("a verified no-op is not an error");
23536
23537        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23538    }
23539
23540    #[test]
23541    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23542        ask_test_home();
23543        let mut runner = runner_at(RunStatus::Implementing);
23544        // Candidate A declares a verified no-op; candidate B simply wrote
23545        // nothing and said nothing about why. One candidate's claim is not
23546        // the whole run's agreement.
23547        set_candidates(
23548            &mut runner,
23549            &[(true, Some("already on main at b32cfc4")), (true, None)],
23550        );
23551
23552        let err = runner
23553            .after_implement()
23554            .expect_err("an unverified empty candidate must still fail the run");
23555
23556        assert!(
23557            err.to_string().contains("no candidate produced a change"),
23558            "{err}"
23559        );
23560        assert_eq!(runner.state.status, RunStatus::Failed);
23561    }
23562
23563    #[test]
23564    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23565        ask_test_home();
23566        let mut runner = runner_at(RunStatus::Implementing);
23567        set_candidates(&mut runner, &[(true, None), (true, None)]);
23568
23569        let err = runner
23570            .after_implement()
23571            .expect_err("no candidate declared anything; this is an ordinary failure");
23572
23573        assert!(
23574            err.to_string().contains("no candidate produced a change"),
23575            "{err}"
23576        );
23577        assert_eq!(runner.state.status, RunStatus::Failed);
23578    }
23579}
23580
23581/// A graft rollback whose restore could not be confirmed (see
23582/// [`Runner::graft`]).
23583#[derive(Debug)]
23584struct GraftRestoreFailed(String);
23585
23586impl std::fmt::Display for GraftRestoreFailed {
23587    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23588        write!(
23589            f,
23590            "could not restore the winner to its pre-graft commit: {}",
23591            self.0
23592        )
23593    }
23594}
23595
23596impl std::error::Error for GraftRestoreFailed {}
23597
23598/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23599/// not the agent: they never draw on an agent's own timeout, and running out of
23600/// them never reads as an agent failure.
23601#[cfg(test)]
23602mod resource_wait_tests {
23603    use super::*;
23604    use std::collections::BTreeMap;
23605
23606    fn sh(id: &str, script: &str) -> AgentSpec {
23607        AgentSpec {
23608            id: id.to_owned(),
23609            kind: crate::config::AgentKind::Command,
23610            model: None,
23611            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23612            extra_args: Vec::new(),
23613            env: BTreeMap::new(),
23614            prompt_delivery: None,
23615            max_turn_secs: None,
23616        }
23617    }
23618
23619    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23620        SeatJob {
23621            seat: SeatState::new("impl-A", &spec.id, 1),
23622            spec,
23623            cwd: dir.to_path_buf(),
23624            prompt: "do it".to_owned(),
23625            timeout,
23626            allow_write: true,
23627            sessions: false,
23628            artifacts: dir.join("art"),
23629            stem: "impl-A".to_owned(),
23630            handover: None,
23631        }
23632    }
23633
23634    fn state(resource_wait_secs: u64) -> RunState {
23635        let mut config = Config::default();
23636        config.graph.timeout_resource_wait = resource_wait_secs;
23637        RunState::new(
23638            PathBuf::from("/repo"),
23639            "main".to_owned(),
23640            "abc1234".to_owned(),
23641            "task".to_owned(),
23642            config,
23643        )
23644    }
23645
23646    fn hold(cache: &Path) -> crate::cache::Guard {
23647        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23648        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23649            crate::cache::AcquireOutcome::Acquired(g) => g,
23650            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23651        }
23652    }
23653
23654    async fn run(
23655        jobs: Vec<SeatJob>,
23656        sem: Arc<Semaphore>,
23657        cache: Option<&Path>,
23658        st: &mut RunState,
23659    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23660        let prompts = Prompts::default();
23661        let ctx = WaveCtx {
23662            run: "r",
23663            node: "implement",
23664            prompts: &prompts,
23665            cache,
23666            round: None,
23667            carry_seats: false,
23668            pauses: None,
23669        };
23670        wave(jobs, sem, &ctx, st, 0).await
23671    }
23672
23673    #[tokio::test]
23674    async fn verification_cache_wait_preserves_the_command_budget() {
23675        crate::run::pin_test_home();
23676        let tmp = tempfile::tempdir().unwrap();
23677        let cache = tmp.path().join("target");
23678        let guard = hold(&cache);
23679        let release = tokio::spawn(async move {
23680            tokio::time::sleep(Duration::from_millis(100)).await;
23681            guard.release();
23682        });
23683        let mut st = state(10);
23684        let budget = Duration::from_millis(50);
23685        let wait = repository_wait(&st, None);
23686        let (outcomes, _) = with_cache_lease(
23687            &mut st,
23688            Some(&cache),
23689            "gate",
23690            "gate",
23691            tmp.path(),
23692            "abc1234",
23693            budget,
23694            "test gate",
23695            &wait,
23696            |_, available| async move {
23697                assert_eq!(
23698                    available, budget,
23699                    "contention must not shorten verification"
23700                );
23701                (Vec::new(), false, Vec::new())
23702            },
23703        )
23704        .await
23705        .unwrap();
23706        release.await.unwrap();
23707        assert!(outcomes.is_empty(), "{outcomes:?}");
23708        assert!(
23709            st.events
23710                .iter()
23711                .any(|event| event.message.contains("waiting for"))
23712        );
23713        assert!(st.resource_wait.is_none());
23714        // The verification body no longer owns the lease after returning.
23715        hold(&cache).release();
23716    }
23717
23718    #[tokio::test]
23719    async fn verification_cache_wait_expires_without_starting_commands() {
23720        crate::run::pin_test_home();
23721        let tmp = tempfile::tempdir().unwrap();
23722        let cache = tmp.path().join("target");
23723        let _held = hold(&cache);
23724        let mut st = state(0);
23725        let wait = repository_wait(&st, None);
23726        let (outcomes, _) = with_cache_lease(
23727            &mut st,
23728            Some(&cache),
23729            "gate",
23730            "gate",
23731            tmp.path(),
23732            "abc1234",
23733            Duration::from_secs(60),
23734            "test gate",
23735            &wait,
23736            |_, _| async { panic!("commands must not start without the cache lease") },
23737        )
23738        .await
23739        .unwrap();
23740        assert_eq!(outcomes.len(), 1);
23741        assert!(outcomes[0].resource_blocked);
23742        assert!(st.resource_wait.is_none());
23743        assert!(matches!(
23744            st.resource_block.unwrap().cause,
23745            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23746        ));
23747    }
23748
23749    #[tokio::test]
23750    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23751        crate::run::pin_test_home();
23752        let tmp = tempfile::tempdir().unwrap();
23753        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23754        let cache = tmp.path().join("cache");
23755        let mut state = state(21600);
23756        let wait = repository_wait(&state, None);
23757        let (outcomes, _) = with_cache_lease(
23758            &mut state,
23759            Some(&cache),
23760            "gate",
23761            "gate",
23762            tmp.path(),
23763            "head",
23764            Duration::from_secs(60),
23765            "gate",
23766            &wait,
23767            |_, _| async { panic!("freshness failure must not start verification") },
23768        )
23769        .await
23770        .unwrap();
23771        assert_eq!(outcomes.len(), 1);
23772        assert!(outcomes[0].resource_blocked);
23773        let block = state.resource_block.unwrap();
23774        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23775        assert!(!block.description().contains("timed out"));
23776    }
23777
23778    #[tokio::test]
23779    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23780        crate::run::pin_test_home();
23781        let tmp = tempfile::tempdir().unwrap();
23782        let cache = tmp.path().join("cache");
23783        let _held = hold(&cache);
23784        let mut state = state(21600);
23785        let pause = Pause::new();
23786        let interrupt = Pause::new();
23787        let cause = ParkCause::Operator;
23788        pause.park_for(cause.clone());
23789        let prompts = Prompts::default();
23790        let ctx = WaveCtx {
23791            run: "test",
23792            node: "fix",
23793            prompts: &prompts,
23794            cache: Some(&cache),
23795            round: None,
23796            carry_seats: false,
23797            pauses: Some((&pause, &interrupt)),
23798        };
23799        let result = wave(
23800            vec![job(
23801                sh("writer", "exit 99"),
23802                tmp.path(),
23803                Duration::from_secs(1),
23804            )],
23805            Arc::new(Semaphore::new(1)),
23806            &ctx,
23807            &mut state,
23808            0,
23809        )
23810        .await;
23811        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23812        assert_eq!(result[0].1.turns, 0);
23813        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23814        assert_eq!(
23815            state.resource_block.unwrap().cause,
23816            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23817        );
23818        assert!(state.active.is_empty());
23819    }
23820
23821    #[tokio::test]
23822    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23823        crate::run::pin_test_home();
23824        let tmp = tempfile::tempdir().unwrap();
23825        let cache = tmp.path().join("target-a");
23826        let guard = hold(&cache);
23827        // Held for longer than the agent's whole 1s budget.
23828        tokio::spawn(async move {
23829            tokio::time::sleep(Duration::from_millis(1500)).await;
23830            guard.release();
23831        });
23832        let mut st = state(60);
23833        let results = run(
23834            vec![job(
23835                sh("alpha", "echo ok"),
23836                tmp.path(),
23837                Duration::from_secs(1),
23838            )],
23839            Arc::new(Semaphore::new(1)),
23840            Some(&cache),
23841            &mut st,
23842        )
23843        .await;
23844        assert!(
23845            matches!(results[0].2, AgentOutcome::Ok(_)),
23846            "the wait must not be charged to the agent"
23847        );
23848        assert!(st.resource_block.is_none());
23849        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23850        assert!(st.resource_wait.is_none() && st.active.is_empty());
23851    }
23852
23853    #[tokio::test]
23854    #[cfg(unix)]
23855    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23856        use std::os::fd::AsRawFd as _;
23857        crate::run::pin_test_home();
23858        let tmp = tempfile::tempdir().unwrap();
23859        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23860        let common = tmp.path().join(".git");
23861        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23862        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23863        assert_eq!(
23864            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23865            0
23866        );
23867        let mut st = state(1);
23868        st.config.coordination = crate::config::test_coordination();
23869        let mut reader = job(
23870            sh("reader", "echo must-not-run"),
23871            tmp.path(),
23872            Duration::from_secs(30),
23873        );
23874        reader.allow_write = false;
23875        let results = run(
23876            vec![reader.clone()],
23877            Arc::new(Semaphore::new(1)),
23878            None,
23879            &mut st,
23880        )
23881        .await;
23882        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23883        assert_eq!(results[0].1.turns, 0);
23884        assert!(FailClass::of(&results[0].2).is_none());
23885        assert_eq!(
23886            st.resource_block.as_ref().unwrap().kind,
23887            "repository coordination"
23888        );
23889        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23890        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23891        drop(external);
23892        let editor = crate::repo_lock::SeatLease::for_checkout(
23893            tmp.path(),
23894            true,
23895            crate::config::test_coordination().as_ref(),
23896        )
23897        .await
23898        .unwrap();
23899        let pause = Pause::new();
23900        let interrupt = Pause::new();
23901        let prompts = Prompts::default();
23902        let ctx = WaveCtx {
23903            run: "r",
23904            node: "review",
23905            prompts: &prompts,
23906            cache: None,
23907            round: None,
23908            carry_seats: false,
23909            pauses: Some((&pause, &interrupt)),
23910        };
23911        let (results, ()) = tokio::join!(
23912            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23913            async {
23914                tokio::time::sleep(Duration::from_millis(50)).await;
23915                pause.park();
23916            }
23917        );
23918        assert!(
23919            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23920        );
23921        assert_eq!(results[0].1.turns, 0);
23922        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23923        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23924        drop(editor);
23925    }
23926
23927    #[tokio::test]
23928    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23929        crate::run::pin_test_home();
23930        let tmp = tempfile::tempdir().unwrap();
23931        let cache = tmp.path().join("target-b");
23932        let _held = hold(&cache);
23933        let mut st = state(1);
23934        let results = run(
23935            vec![job(
23936                sh("alpha", "echo ok"),
23937                tmp.path(),
23938                Duration::from_secs(1),
23939            )],
23940            Arc::new(Semaphore::new(1)),
23941            Some(&cache),
23942            &mut st,
23943        )
23944        .await;
23945        let (_, seat, out) = &results[0];
23946        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23947        assert!(wave_blocked(&results));
23948        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23949        assert_eq!(seat.turns, 0, "nothing was asked");
23950        assert!(
23951            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23952            "no agent was spawned"
23953        );
23954        assert!(FailClass::of(out).is_none(), "not a failure class");
23955        let block = st.resource_block.as_ref().expect("recorded for the resume");
23956        assert_eq!(block.kind, "shared build cache");
23957        assert!(matches!(
23958            block.cause,
23959            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23960        ));
23961        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23962        assert!(st.active.is_empty());
23963    }
23964
23965    #[tokio::test]
23966    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23967        crate::run::pin_test_home();
23968        let tmp = tempfile::tempdir().unwrap();
23969        let mut st = state(1);
23970        let results = run(
23971            vec![job(
23972                sh("alpha", "echo ok"),
23973                tmp.path(),
23974                Duration::from_secs(1),
23975            )],
23976            Arc::new(Semaphore::new(0)),
23977            None,
23978            &mut st,
23979        )
23980        .await;
23981        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23982        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23983        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23984    }
23985
23986    #[tokio::test]
23987    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23988        crate::run::pin_test_home();
23989        let tmp = tempfile::tempdir().unwrap();
23990        let mut st = state(60);
23991        let results = run(
23992            vec![job(
23993                sh("alpha", "sleep 30"),
23994                tmp.path(),
23995                Duration::from_secs(1),
23996            )],
23997            Arc::new(Semaphore::new(1)),
23998            None,
23999            &mut st,
24000        )
24001        .await;
24002        let out = &results[0].2;
24003        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24004        let class = FailClass::of(out).expect("a failure");
24005        assert_eq!(class, FailClass::Timeout);
24006        assert!(should_hand_over(None, &class), "existing handover policy");
24007        assert!(st.resource_block.is_none());
24008    }
24009}