Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Recorded before the first save, so a crash right after minting
773        // cannot leave a run with no origin. Legibility only: nothing reads it
774        // to decide anything.
775        state.fetched_base = Some(fetched_base);
776        state.origin = Some(origin);
777        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778        state.status = RunStatus::Prep;
779        state.event("start", format!("run {} created", state.id));
780        startup.finish(&mut state, Ok(()))?;
781        Ok(Self {
782            state,
783            roles,
784            sem: Arc::new(Semaphore::new(max_parallel)),
785            pause: Pause::new(),
786            interrupt: Pause::new(),
787        })
788    }
789
790    /// Open a review-only run against work that already exists on `branch`.
791    ///
792    /// The expensive half of the graph is the implement wave — measured at
793    /// 111 and 134 internal tool-loop turns on this repository, against a
794    /// handful for a judge or a reviewer. The cheap half is worth running on
795    /// hand-written work too, and there was no way to reach it.
796    ///
797    /// No new state and no schema change are needed: a run with **one** viable
798    /// candidate and a tally already decided degrades `execute` to exactly
799    /// review → gate → merge, because `judge` skips a single-candidate field,
800    /// `deliberate` has fewer than two first choices to reconcile, `vote`
801    /// returns early, `tally` is already present and `fold_losers` has no
802    /// losers. Resuming such a run therefore does the right thing as well.
803    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804        Self::review_taking_over(repo, branch, config, None, origin).await
805    }
806
807    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
808    /// at the same task still has `branch` checked out, its worktree is
809    /// released first if that is safe (see [`crate::handover`]), and the
810    /// review refuses with the reason if it is not. `None` is a hand-run
811    /// review: it has no earlier attempts, so only a worktree of a dead run
812    /// magi recorded itself can be released.
813    pub async fn review_taking_over(
814        repo: &Path,
815        branch: &str,
816        config: Config,
817        takeover: Option<crate::handover::Takeover>,
818        origin: Origin,
819    ) -> Result<Self> {
820        let repo = git::toplevel(repo).await?;
821        let missing = agent::missing_programs(&config.agents);
822        if !missing.is_empty() {
823            bail!(
824                "these agent programs are not on PATH: {}. Fix the roster in \
825                 magi.toml or install them.",
826                missing.join(", ")
827            );
828        }
829        let base_branch =
830            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831                .await?;
832        if base_branch == branch {
833            bail!("`{branch}` is the base branch; there is nothing to review against");
834        }
835        let fetched_base = resolve_base(
836            &repo,
837            &base_branch,
838            &config.merge.remote,
839            config.coordination.as_ref(),
840        )
841        .await?;
842        let base_commit = fetched_base.commit.clone();
843
844        let roles = config.resolve_roles()?;
845        let max_parallel = config.graph.max_parallel.max(1);
846        let mut state = RunState::new(
847            repo.clone(),
848            base_branch,
849            base_commit.clone(),
850            format!("Review the work already on branch `{branch}`."),
851            config,
852        );
853        state.fetched_base = Some(fetched_base);
854        state.origin = Some(origin);
855        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857        let opened = async {
858            startup.stage(&mut state, "handover")?;
859
860            // Released before anything else touches the branch: a stale local
861            // branch is moved with `git branch -f`, which git refuses while an
862            // earlier attempt's worktree still has it checked out. Everything
863            // after this point that can fail puts the old run back.
864            // A hand-run review has no task, hence no earlier attempts, but a
865            // worktree of a dead run magi made may still be released.
866            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867                earlier: Vec::new(),
868                home: crate::run::home(),
869                choice: None,
870                task_instruction: None,
871            });
872            let wait = repository_wait(&state, None);
873            let coordination_config = state.config.coordination.clone();
874            let coordination =
875                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876                    .await?;
877            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878            if let Some(released) = &released {
879                state.event(
880                    "release",
881                    format!(
882                        "took `{branch}` over from run {}: its worktree was released: {}",
883                        crate::run::short_of(&released.old_id),
884                        released.audit
885                    ),
886                );
887            }
888            let opened = async {
889                startup.stage(&mut state, "review-preflight")?;
890                // The owner's answer to an earlier divergence question is applied
891                // here: after the release (git will not move a checked-out branch)
892                // and before the sync that would otherwise ask again.
893                state.foreign_checkout = crate::handover::preflight_review(
894                    &repo,
895                    branch,
896                    &takeover.home,
897                    &state.config.merge.remote,
898                )
899                .await?;
900                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902                }
903                if let Some(choice) = takeover.choice.as_ref()
904                    && let Err(e) =
905                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906                            .await
907                {
908                    return Err(e.context("applying the owner's answer about the diverged branch"));
909                }
910                // An earlier attempt of this task that published the task head, and
911                // whose work is the branch under review, hands that head over: the
912                // review run updates the same pull request. Without one (a legacy
913                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
914                // own head and its own open pull request is adopted as before.
915                for id in takeover.earlier.iter().rev() {
916                    if let Ok(old) = RunState::load_under(id, &takeover.home)
917                        && let Some(head) = old.pr_head.clone()
918                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919                    {
920                        state.pr_head = Some(head);
921                        break;
922                    }
923                }
924                drop(coordination);
925                startup.stage(&mut state, "open-review")?;
926                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927            }
928            .await;
929            if opened.is_err()
930                && let Some(released) = &released
931            {
932                // Restoration is ownership checked and best effort. Keep the original
933                // startup cause even if reacquiring coordination is no longer possible.
934                match crate::repo_lock::SeatLease::operation_with(
935                    &repo,
936                    coordination_config.as_ref(),
937                    &wait,
938                )
939                .await
940                {
941                    Ok(_coordination) => {
942                        released.restore(&repo, branch).await;
943                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
944                            .map(|old| {
945                                format!(
946                                    "predecessor released_to={:?}; candidate path exists={}",
947                                    old.released_to,
948                                    old.candidates.iter().any(|c| {
949                                        c.branch == branch && c.worktree.exists()
950                                    }),
951                                )
952                            })
953                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955                    }
956                    Err(error) => {
957                        tracing::warn!(
958                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959                            state.id
960                        );
961                        state.event(
962                            "startup-restore",
963                            format!("restoration not attempted: {error:#}"),
964                        );
965                    }
966                }
967            }
968            opened
969        }
970        .await;
971        startup.finish(&mut state, opened)?;
972        Ok(Self {
973            state,
974            roles,
975            sem: Arc::new(Semaphore::new(max_parallel)),
976            pause: Pause::new(),
977            interrupt: Pause::new(),
978        })
979    }
980
981    /// Recover exactly the operator-selected public implementation, without touching old sources.
982    pub async fn recover_pr(
983        repo: &Path,
984        task: &str,
985        selection: crate::pr_recovery::Selection,
986        mut config: Config,
987    ) -> Result<Self> {
988        selection.configure(&mut config)?;
989        selection.check_task(task, &selection.instruction)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.origin = Some(crate::run::Origin::queue(task));
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize exactly one additional rebase, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true).await
1357    }
1358
1359    async fn authorize_base_recovery_request(
1360        home: &Path,
1361        id: &str,
1362        task: &str,
1363        resumption: bool,
1364    ) -> Result<()> {
1365        let _owner = crate::pr_head::owner_lock(home, id)?;
1366        let mut state = RunState::load_under(id, home)?;
1367        crate::pr_head::ensure_owner(&state)?;
1368        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369        anyhow::ensure!(
1370            queued.runs.last().is_some_and(|run| run == id)
1371                && queued.status != crate::queue::TaskStatus::Done
1372                && !queued.fresh_start
1373                && queued.review_branch.is_none(),
1374            "run task ownership changed"
1375        );
1376        anyhow::ensure!(
1377            state
1378                .origin
1379                .as_ref()
1380                .and_then(|o| o.task.as_deref())
1381                .is_none_or(|owner| owner == task),
1382            "run belongs to another task"
1383        );
1384        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385        anyhow::ensure!(
1386            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387            "run is live or its ownership is unknown"
1388        );
1389        // Judge the previous driver's lease before acquiring our maintenance
1390        // lease. The owner lock excludes new driver claims across both steps.
1391        let _execution = crate::driver::maintenance(home, id)?;
1392        anyhow::ensure!(
1393            state.status.resumable()
1394                && state.released_to.is_none()
1395                && state.retired_to.is_none()
1396                && state.pr_taken_over_by.is_none(),
1397            "run ownership is final"
1398        );
1399        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400        anyhow::ensure!(
1401            !state.pr_publication_pending
1402                && !crate::handover::landing_checkpoint(&state)
1403                && state.merge_decision.is_none(),
1404            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405        );
1406        let winner = state.winner().context("run has no retained winner")?;
1407        anyhow::ensure!(
1408            git::is_clean(&winner.worktree).await?,
1409            "retained worktree has uncommitted changes"
1410        );
1411        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412        anyhow::ensure!(
1413            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414            "retained worktree does not match the winner branch"
1415        );
1416        state.ensure_base_episode();
1417        state.record_base_sync_stop(&head);
1418        state.save()?;
1419        let fetched =
1420            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421        anyhow::ensure!(
1422            fetched.ok(),
1423            "cannot authorize base recovery without a fetched base"
1424        );
1425        let approved_tip = git::rev_parse(
1426            &state.repo,
1427            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428        )
1429        .await?;
1430        if resumption {
1431            anyhow::ensure!(
1432                crate::daemon::compatible_run_instruction(&state, &queued),
1433                "resumption instruction does not match the retained run"
1434            );
1435            let approval = queued
1436                .resumption
1437                .as_ref()
1438                .context("no operator resumption")?;
1439            anyhow::ensure!(
1440                approval.instruction == queued.instruction,
1441                "resumption approval does not match the current task instruction"
1442            );
1443            state.authorize_base_sync_resumption(task, approval)?;
1444        } else {
1445            state.authorize_base_sync(task)?;
1446        }
1447        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448        state
1449            .base_sync_recovery
1450            .authorizations
1451            .last_mut()
1452            .unwrap()
1453            .scope = Some(crate::base_sync_recovery::Scope {
1454            tip: approved_tip,
1455            instruction: queued.instruction.clone(),
1456            episode,
1457        });
1458        state.event(
1459            "base-recovery",
1460            "owner authorized a bounded base synchronization interval; fresh verification required",
1461        );
1462        state.save()?;
1463        Ok(())
1464    }
1465
1466    /// Reopen an existing run.
1467    pub fn resume(id: &str) -> Result<Self> {
1468        let state = RunState::load(id)?;
1469        if state.startup.as_ref().is_some_and(|s| !s.complete)
1470            || (state.status == RunStatus::Prep
1471                && state.instruction.trim().is_empty()
1472                && state.candidates.is_empty()
1473                && state.prep_complete == Some(false))
1474        {
1475            bail!(
1476                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477            );
1478        }
1479        crate::pr_head::ensure_owner(&state)?;
1480        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481            bail!(
1482                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483            );
1484        }
1485        if let Some(to) = &state.released_to {
1486            bail!(
1487                "run {} cannot be resumed: its worktree was released to run {}",
1488                state.short(),
1489                crate::run::short_of(to)
1490            );
1491        }
1492        let roles = state.config.resolve_roles()?;
1493        if let Some(id) = &state.config.graph.selected_implementer
1494            && state.candidates.iter().any(|c| c.agent != *id)
1495        {
1496            bail!(
1497                "replacement implementer pin does not match the saved candidates; task must stay held"
1498            );
1499        }
1500        let max_parallel = state.config.graph.max_parallel.max(1);
1501        Ok(Self {
1502            state,
1503            roles,
1504            sem: Arc::new(Semaphore::new(max_parallel)),
1505            pause: Pause::new(),
1506            interrupt: Pause::new(),
1507        })
1508    }
1509
1510    /// Walk the graph to a terminal state, skipping nodes already recorded.
1511    ///
1512    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1513    /// the phone - ends here, so this is the one place a run that ended
1514    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1515    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1516    pub async fn execute(&mut self) -> Result<()> {
1517        anyhow::ensure!(
1518            self.state.startup.as_ref().is_none_or(|s| s.complete),
1519            "startup incomplete; inspect retained evidence before explicit recovery"
1520        );
1521        if let Some(selection) = &self.state.pr_recovery
1522            && (selection.source_proof.is_some() || selection.source_run.is_some())
1523        {
1524            Box::pin(selection.verify_source(&self.state.repo)).await?;
1525        }
1526        // Queue claims cover daemon work; this also serializes manual drivers
1527        // consuming a repair answer, including the new review and decision.
1528        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529            || self.state.land_approval.is_some()
1530            || self.state.merge_decision.is_some()
1531            || !self.state.merge_repairs.is_empty()
1532            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533        {
1534            let claim = FixClaim::acquire(&self.state.dir())?;
1535            let disk = RunState::load(&self.state.id)?;
1536            let checkpoint = |s: &RunState| {
1537                (
1538                    s.merge_repairs.len(),
1539                    s.merge_repairs
1540                        .last()
1541                        .map(|r| (r.question.clone(), r.phase)),
1542                    s.land_approval
1543                        .as_ref()
1544                        .map(|a| (a.question.clone(), a.head.clone())),
1545                )
1546            };
1547            if checkpoint(&disk) != checkpoint(&self.state) {
1548                bail!(
1549                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550                );
1551            }
1552            Some(claim)
1553        } else {
1554            None
1555        };
1556        let mut execution = self.claim_driver().await?;
1557        let runner = &mut *execution.runner;
1558        let wait = runner.repository_wait();
1559        // Keep the graph's large future off the callers' stack, including
1560        // standalone task attempts running on a default-sized test thread.
1561        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562        let result = match result {
1563            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564                record_repository_block(&mut runner.state, &e);
1565                runner.park_here()?;
1566                Ok(())
1567            }
1568            other => other,
1569        };
1570        execution.driver.finish(&mut runner.state).await?;
1571        let ended = if result.is_err() {
1572            Some({
1573                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574                if let Err(error) = &result {
1575                    notice.message = format!(
1576                        "Run {} stopped with an error: {}",
1577                        runner.state.id,
1578                        error.to_string().lines().next().unwrap_or("unknown error")
1579                    );
1580                }
1581                notice
1582            })
1583        } else {
1584            crate::notices::run_ended(&runner.state)
1585        };
1586        if let Some(notice) = ended {
1587            crate::notices::raise(notice);
1588        }
1589        result
1590    }
1591
1592    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594    }
1595
1596    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597        let wait = self.repository_wait();
1598        coordinate_operation(&mut self.state, cwd, &wait).await
1599    }
1600
1601    /// Validate ownership and persist the driver claim in one critical section.
1602    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603        let owner = match crate::pr_head::owner_lock_wait(
1604            &crate::run::home(),
1605            &self.state.id,
1606            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607        )
1608        .await
1609        {
1610            Ok(owner) => owner,
1611            Err(error) => {
1612                let detail = format!("worktree maintenance or ownership update: {error:#}");
1613                // Admission did not own run.json: retain this resource outcome
1614                // in memory for queue settlement without overwriting a concurrent
1615                // retirement or publication transaction.
1616                self.state.resource_block = Some(crate::run::ResourceBlock {
1617                    cause: crate::run::ResourceBlockCause::Other,
1618                    node: "start".into(),
1619                    kind: "worktree maintenance".into(),
1620                    detail: detail.clone(),
1621                    at: Timestamp::now(),
1622                });
1623                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624            }
1625        };
1626        let disk = RunState::load(&self.state.id)?;
1627        crate::pr_head::ensure_owner(&disk)?;
1628        let claimed =
1629            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630        // Only proven-live ownership refuses here. Unknown records (legacy runs
1631        // with no claim marker or process identity) fall through to the
1632        // exclusive execution lease taken just below, which is what actually
1633        // keeps a second driver out.
1634        anyhow::ensure!(
1635            disk.liveness(claimed) != crate::run::Liveness::Live,
1636            "run already has a live driver; wait for its execution to end"
1637        );
1638        if let Some(to) = &disk.released_to {
1639            bail!(
1640                "run {} cannot continue: its worktree was released to run {}",
1641                self.state.short(),
1642                crate::run::short_of(to)
1643            );
1644        }
1645        anyhow::ensure!(
1646            serde_json::to_value(&disk.base_sync_recovery)?
1647                == serde_json::to_value(&self.state.base_sync_recovery)?,
1648            "base recovery changed since this run was loaded; reload the saved run"
1649        );
1650        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651        let mut execution = DriverExecution {
1652            runner: self,
1653            driver,
1654        };
1655        // Declare after the execution guard: on any claim error this short
1656        // lock must drop before that guard reloads and releases ownership.
1657        let _owner = owner;
1658        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659            .await?;
1660        let state = &mut execution.runner.state;
1661        // Moving again, so it is no longer parked. Set before the walk rather
1662        // than in `resume`, so every way of re-entering the graph clears it
1663        // and a card cannot claim a run is waiting to be resumed while the
1664        // agents are already working.
1665        state.parked = false;
1666        state.resource_wait = None;
1667        state.resource_block = None;
1668        // Any seat this state still lists as answering belongs to whatever
1669        // process last drove this run — this one included, if it crashed
1670        // mid-wave. Cleared and flushed immediately, before anything else
1671        // runs, so a resume can never show a seat as live when nothing is
1672        // asking it anything yet; the node that actually dispatches the next
1673        // wave repopulates it.
1674        state.clear_active();
1675        // Recorded in the same spot, and flushed together with the clear
1676        // above: this is the pid a reader checks (`RunState::liveness`) when
1677        // no daemon claim exists to answer "is a process still driving this
1678        // run" — a plain `magi run` / `magi review` typed into a terminal
1679        // claims nothing there. Always overwritten, never only-if-absent, so
1680        // a resumed run's stale pid from a previous, possibly-dead process
1681        // can never survive into this one's own report. Unlike
1682        // `clear_active`, this changes on every single `execute()` call, so
1683        // the save below is now unconditional rather than only-if-cleared.
1684        //
1685        // `driver_started_at` is recorded in the same breath, from this same
1686        // pid, so `liveness` can tell a live pid that is genuinely still us
1687        // apart from one the OS has since handed to an unrelated process —
1688        // see that field's own doc for why the pid alone is not enough.
1689        let pid = std::process::id();
1690        state.driver_pid = Some(pid);
1691        state.driver_started_at = crate::proc::process_started_at(pid);
1692        state.driver_exited = false;
1693        state.driver_claimed = Some(true);
1694        state.begin_pass();
1695        execution.driver.bind(state);
1696        state.save()?;
1697        execution.driver.record(state)?;
1698        Ok(execution)
1699    }
1700
1701    /// A grant owns only these repair attempts. Confirmation reviews cannot
1702    /// create another fixer allowance, and every operation is restart-checkpointed.
1703    async fn execute_granted_repair(&mut self) -> Result<()> {
1704        use crate::repair_budget::{Kind, Phase};
1705        let index = self.state.repair_grants.len() - 1;
1706        let winner = self
1707            .state
1708            .winner()
1709            .cloned()
1710            .context("repair winner missing")?;
1711        loop {
1712            let grant = self.state.repair_grants[index].clone();
1713            match grant.phase {
1714                Phase::Pending => {
1715                    if grant.spent >= grant.receipt.request.rounds {
1716                        self.state.repair_grants[index].phase = Phase::Complete;
1717                        self.state.status = RunStatus::Blocked;
1718                        self.state.save()?;
1719                        return Ok(());
1720                    }
1721                    // Reserve before dispatch. A crash can consume an attempt,
1722                    // but can never duplicate it or reset saved fixer history.
1723                    self.state.repair_grants[index].spent += 1;
1724                    self.state.repair_grants[index].phase = Phase::Fixing;
1725                    self.state.save()?;
1726                    match grant.receipt.request.kind {
1727                        Kind::Gate => {
1728                            let failed = self.state.gate.clone();
1729                            let _ = self.gate_fix_round(&winner, &failed).await?;
1730                        }
1731                        Kind::Review => {
1732                            let mut record = self
1733                                .state
1734                                .reviews
1735                                .last()
1736                                .cloned()
1737                                .context("review failure missing")?;
1738                            let round = record.round;
1739                            let step = self.run_fix_round(&winner, &mut record).await?;
1740                            self.state.reviews[round - 1] = record;
1741                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1742                                self.state.repair_grants[index].phase = Phase::Complete;
1743                                self.state.status = RunStatus::Blocked;
1744                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1745                                self.state.save()?;
1746                                return Ok(());
1747                            }
1748                            if matches!(step, FixStep::Resource) {
1749                                self.stop_for_resource()?;
1750                                return Ok(());
1751                            }
1752                            self.state.save()?;
1753                        }
1754                    }
1755                    if self.resource_halt()? {
1756                        return Ok(());
1757                    }
1758                }
1759                Phase::Fixing => {
1760                    // Interrupted fixers may have left source changes or an
1761                    // unpushed commit. Preserve them before fresh assessment.
1762                    let _coordination = self.coordinate(&winner.worktree).await?;
1763                    let rescued = git::rescue_commit(
1764                        &winner.worktree,
1765                        "magi: preserve interrupted granted repair",
1766                    )
1767                    .await?;
1768                    self.state.note_withheld("repair-grant", &rescued.withheld);
1769                    // Fresh checks must supersede even a same-head failure.
1770                    // Archive rather than erase failure evidence or counters.
1771                    self.state
1772                        .verification_history
1773                        .push(crate::run::VerificationEvidence {
1774                            superseded_by: None,
1775                            archived_at: Some(Timestamp::now()),
1776                            fixes_spent: Some(self.state.gate_fixes.len()),
1777                            gate_head: self.state.gate_head.take(),
1778                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1779                            gate: std::mem::take(&mut self.state.gate),
1780                            e2e: self.state.head_e2e.take(),
1781                        });
1782                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1783                        self.state
1784                            .verification_history
1785                            .push(crate::run::VerificationEvidence {
1786                                superseded_by: None,
1787                                archived_at: Some(Timestamp::now()),
1788                                fixes_spent: Some(self.state.gate_fixes.len()),
1789                                gate_head: None,
1790                                gate_ran: false,
1791                                gate: Vec::new(),
1792                                e2e: Some(e2e),
1793                            });
1794                    }
1795                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1796                    self.state.repair_grants[index].head =
1797                        git::rev_parse(&winner.worktree, "HEAD").await?;
1798                    self.state.repair_grants[index].phase = Phase::Reviewing;
1799                    self.state.save()?;
1800                }
1801                Phase::Reviewing => {
1802                    self.review_loop_pass().await?;
1803                    if self.resource_halt()? {
1804                        return Ok(());
1805                    }
1806                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1807                    let clean = self.state.reviews.len() > grant.review_start
1808                        && self.state.reviews.last().is_some_and(|r| {
1809                            r.head == head
1810                                && r.clean
1811                                && r.expected > 0
1812                                && r.answered == r.expected
1813                                && matches!(
1814                                    r.e2e_status(),
1815                                    E2eStatus::Passed | E2eStatus::NotConfigured
1816                                )
1817                        });
1818                    if !clean {
1819                        // A confirmation review carries no additional fix budget.
1820                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1821                            == Kind::Review
1822                            && grant.spent < grant.receipt.request.rounds
1823                            && self.state.reviews.last().is_some_and(|r| {
1824                                r.fix.is_none()
1825                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1826                            }) {
1827                            Phase::Pending
1828                        } else {
1829                            Phase::Complete
1830                        };
1831                        self.state.status = RunStatus::Blocked;
1832                        self.state.event(
1833                            "repair-grant",
1834                            "fresh review did not clear the repaired head; no publication",
1835                        );
1836                        self.state.save()?;
1837                        if self.state.repair_grants[index].phase == Phase::Pending {
1838                            continue;
1839                        }
1840                        return Ok(());
1841                    }
1842                    self.state.repair_grants[index].phase = Phase::Gating;
1843                    self.state.save()?;
1844                }
1845                Phase::Gating => {
1846                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1847                    self.state.reconcile_verification_head(&head);
1848                    let wait = self.repository_wait();
1849                    if self
1850                        .state
1851                        .reviews
1852                        .last()
1853                        .is_none_or(|r| r.head != head || !r.clean)
1854                    {
1855                        self.state.status = RunStatus::Blocked;
1856                        self.state.repair_grants[index].phase = Phase::Complete;
1857                        self.state.event(
1858                            "repair-grant",
1859                            "head changed since fresh review; repair held",
1860                        );
1861                        self.state.save()?;
1862                        return Ok(());
1863                    }
1864                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1865                        return Ok(());
1866                    }
1867                    let outcomes = self.run_gate(&winner).await?;
1868                    if verify_inconclusive(&outcomes) {
1869                        return Ok(());
1870                    }
1871                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1872                    if after != head {
1873                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1874                        self.state.repair_grants[index].head = after;
1875                        self.state.repair_grants[index].phase = Phase::Reviewing;
1876                        self.state.status = RunStatus::Blocked;
1877                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1878                        self.state.save()?;
1879                        return Ok(());
1880                    }
1881                    let passed = outcomes.iter().all(CommandOutcome::ok);
1882                    self.state.repair_grants[index].succeeded = passed;
1883                    self.state.gate = outcomes;
1884                    self.state.gate_ran = true;
1885                    self.state.status = if passed {
1886                        RunStatus::Gating
1887                    } else {
1888                        RunStatus::Blocked
1889                    };
1890                    self.state.repair_grants[index].phase = if !passed
1891                        && grant.receipt.request.kind == Kind::Gate
1892                        && grant.spent < grant.receipt.request.rounds
1893                    {
1894                        Phase::Pending
1895                    } else {
1896                        Phase::Complete
1897                    };
1898                    if passed && self.state.config.merge.mode == MergeMode::Pr {
1899                        // Persist publication debt with the successful checkpoint.
1900                        self.state.pr_publication_pending = true;
1901                    } else if passed {
1902                        self.state.status = RunStatus::Ready;
1903                    }
1904                    self.state.save()?;
1905                    if !passed {
1906                        continue;
1907                    }
1908                    // Existing PR ownership and push lease stay in place. The
1909                    // normal publisher owes the new reviewed/gated SHA, and
1910                    // land's fresh owner approval remains mandatory.
1911                    if self.state.config.merge.mode != MergeMode::Pr {
1912                        self.state.status = RunStatus::Ready;
1913                        self.state.event(
1914                            "repair-grant",
1915                            "repair verified locally; grant carries no merge authorization",
1916                        );
1917                        self.state.save()?;
1918                        return Ok(());
1919                    }
1920                    return self.merge().await;
1921                }
1922                Phase::Complete => return Ok(()),
1923            }
1924        }
1925    }
1926
1927    async fn execute_graph(&mut self) -> Result<()> {
1928        self.state.configure_recovery_resume()?;
1929        if self
1930            .state
1931            .repair_grants
1932            .last()
1933            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1934        {
1935            return self.execute_granted_repair().await;
1936        }
1937        if self
1938            .state
1939            .repair_grants
1940            .last()
1941            .is_some_and(|g| !g.succeeded)
1942        {
1943            self.state.status = RunStatus::Blocked;
1944            self.state.save()?;
1945            return Ok(());
1946        }
1947        // Preservation may invalidate an old empty classification. Resolve it
1948        // before any resume shortcut can reuse a verdict or publish a branch.
1949        self.persist_candidates().await?;
1950        // A saved base stop outranks stage markers, including legacy Voting.
1951        // Recovery never reenters implementation, deliberation or voting.
1952        if self.state.base_sync_blocked() {
1953            self.sync_to_base().await?;
1954            if !self.state.base_sync_blocked()
1955                && self.state.status != RunStatus::AlreadyInBase
1956                && !self.resource_halt()?
1957            {
1958                self.finish_after_tally().await?;
1959            }
1960            return Ok(());
1961        }
1962        if self
1963            .state
1964            .merge_decision
1965            .as_ref()
1966            .is_some_and(|decision| !decision.closed)
1967            && self.state.status != RunStatus::Landing
1968        {
1969            if crate::merge_decision::resumable_hold(&self.state) {
1970                self.state.merge = Some(MergeOutcome {
1971                    mode: MergeMode::Pr,
1972                    ok: false,
1973                    detail: self
1974                        .state
1975                        .merge_decision
1976                        .as_ref()
1977                        .unwrap()
1978                        .target
1979                        .pr
1980                        .clone(),
1981                    empty: false,
1982                });
1983                self.run_land().await?;
1984                self.settle_questions();
1985                return Ok(());
1986            }
1987            bail!(
1988                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1989            );
1990        }
1991        if self.state.config.graph.selected_implementer.is_some()
1992            && self.state.status == RunStatus::Blocked
1993            && self.state.tally.is_none()
1994            && self.state.candidate_preservation.is_empty()
1995            && self.state.interrupted_implementations.is_empty()
1996        {
1997            bail!(
1998                "selected implementation is held; explicitly release the owning task to retry it"
1999            );
2000        }
2001
2002        if self.state.merge_repairs.last().is_some_and(|r| {
2003            matches!(
2004                r.phase,
2005                crate::run::MergeRepairPhase::Requested
2006                    | crate::run::MergeRepairPhase::Fixing
2007                    | crate::run::MergeRepairPhase::Reviewing
2008            )
2009        }) {
2010            if self.repair_merge_review().await? {
2011                self.run_land().await?;
2012            }
2013            self.settle_questions();
2014            return Ok(());
2015        }
2016        // Publication already passed review and gate. Resume only that node:
2017        // syncing to a newer base here could change the verified commit.
2018        if self.state.pr_publication_pending {
2019            self.merge().await?;
2020            if self.resource_halt()? {
2021                return Ok(());
2022            }
2023            self.state.save()?;
2024            self.settle_questions();
2025            return Ok(());
2026        }
2027        // A run that already lost its quorum never resumes into the verdict
2028        // machinery: `deliberate` and `vote` would otherwise clobber the
2029        // stalled marker back to Voting and the run would keep going past a
2030        // verdict that is no longer trustworthy. Everything already recorded is
2031        // kept, so the run stays resumable (or foldable) for a human to pick up.
2032        //
2033        // On --resume the run gets one chance to repair itself: the seats a
2034        // rate limit took out are re-asked. If their quota has since reset and
2035        // the quorum is restored, the run picks up and finishes; otherwise it
2036        // stays stale and still-resumable for a later retry. If it does not
2037        // recover, the returned status stays `Stalled` and nothing was
2038        // clobbered (the recovery only mutates entries for the lost seats).
2039        if self.state.status == RunStatus::Stalled {
2040            if self.recover_stall().await? {
2041                self.finish_after_tally().await?;
2042            } else {
2043                // Still below quorum: persist the marker and stay resumable.
2044                self.state.save()?;
2045            }
2046            return Ok(());
2047        }
2048        // A run parked inside `land` - watching CI, mid fix-round, or
2049        // waiting on the owner's merge approval - resumes directly into it,
2050        // never back through `prep`. Everything before `merge` already
2051        // concluded; that is the only way `status` reaches `Landing` in the
2052        // first place. Re-walking `review_loop` first would also be actively
2053        // wrong: its own status recomputation (see its doc) treats any
2054        // clean round as reason to set `status` to `Gating`, which would
2055        // clobber this marker before `merge` ever ran, and this run would
2056        // never find its way back into `land` at all.
2057        if self.state.status == RunStatus::Landing
2058            || crate::handover::landing_checkpoint(&self.state)
2059        {
2060            self.state.land_sync_pending = true;
2061            self.state.save()?;
2062            self.run_land().await?;
2063            // `run_land` may have settled the run right here - CI came back
2064            // green and the PR merged, say - without ever passing back
2065            // through `merge`'s own trailing call. Whatever it left `status`
2066            // as is what this has to read.
2067            self.settle_questions();
2068            return Ok(());
2069        }
2070        // A run parked at `merge` on the owner's word about a withheld PR text
2071        // resumes straight into `merge`. Re-walking the tail would sync to the
2072        // base again (possibly rebasing, possibly spending the fixer) after the
2073        // gate had passed, and then push a tree nobody reviewed or gated.
2074        if self.state.github_text.is_some()
2075            && self.state.merge.is_none()
2076            && self.state.status == RunStatus::Gating
2077        {
2078            self.merge().await?;
2079            if self.resource_halt()? {
2080                return Ok(());
2081            }
2082            self.state.save()?;
2083            return Ok(());
2084        }
2085        if let Err(e) = self.prep().await {
2086            if e.is::<crate::repo_lock::ResourceBlocked>() {
2087                return Err(e);
2088            }
2089            self.state.status = RunStatus::Blocked;
2090            self.state
2091                .prep_failure
2092                .get_or_insert_with(|| format!("{e:#}"));
2093            self.state
2094                .event("prep", format!("preparation stopped: {e:#}"));
2095            self.state.save()?;
2096            return Err(e);
2097        }
2098        if self.park_here()? {
2099            return Ok(());
2100        }
2101        self.advise().await?;
2102        if self.park_here()? {
2103            return Ok(());
2104        }
2105        self.implement().await?;
2106        if self.park_here()? {
2107            return Ok(());
2108        }
2109        // `after_implement` already saved the state and settled any open
2110        // questions when it set this; nothing later in the graph has
2111        // anything to judge.
2112        if self.state.status == RunStatus::VerifiedNoop {
2113            return Ok(());
2114        }
2115        self.judge().await?;
2116        if self.park_here()? {
2117            return Ok(());
2118        }
2119        self.deliberate().await?;
2120        if self.park_here()? {
2121            return Ok(());
2122        }
2123        self.vote().await?;
2124        if self.park_here()? {
2125            return Ok(());
2126        }
2127        self.tally()?;
2128        // A verdict that lost its quorum is not trustworthy: do not review,
2129        // gate, or merge on it. Everything already done is kept, so the run
2130        // stays resumable (or foldable); the human can replace the agent that
2131        // ran out of quota and pick it up.
2132        if self.state.status == RunStatus::Stalled {
2133            // Persist the stalled marker now — the normal end-of-execute save
2134            // below is below this early return, and without it a resumed run
2135            // would reload a pre-tally status and keep going.
2136            self.state.save()?;
2137            return Ok(());
2138        }
2139        self.finish_after_tally().await?;
2140        Ok(())
2141    }
2142
2143    /// A wave could not start its agents because a shared resource stayed busy
2144    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2145    /// resumable. Unstarted seats have no result; completed peers retain
2146    /// their saved candidate state.
2147    fn stop_for_resource(&mut self) -> Result<()> {
2148        // Resource acquisition observes cancellation inside a node. Complete
2149        // the same durable park as every other node boundary before returning.
2150        if self.pause.parked() || self.interrupt.parked() {
2151            self.park_here()?;
2152            return Ok(());
2153        }
2154        if self.state.resource_block.is_some() {
2155            self.state.status = RunStatus::Blocked;
2156        }
2157        self.state.save()
2158    }
2159
2160    /// Stop walking the graph if a node ended on a resource block.
2161    fn resource_halt(&mut self) -> Result<bool> {
2162        if self.state.resource_block.is_none() {
2163            return Ok(false);
2164        }
2165        self.stop_for_resource()?;
2166        Ok(true)
2167    }
2168
2169    /// Park here if asked to, recording it in the run's own timeline.
2170    ///
2171    /// Returns whether the caller should stop walking the graph. The state is
2172    /// saved either way by the node that just finished; this adds the event so
2173    /// the operator's card says why a run that is neither finished nor moving
2174    /// is sitting where it is.
2175    fn park_here(&mut self) -> Result<bool> {
2176        let parking = self.pause.parked() || self.interrupt.parked();
2177        if !parking && self.state.resource_block.is_some() {
2178            self.stop_for_resource()?;
2179            return Ok(true);
2180        }
2181        // Either handle asking is enough - see `Pause`'s own doc for why
2182        // they are never the same one. `interrupt` is checked second so a
2183        // reason it carries is preferred in the message below over a plain
2184        // shutdown park racing it at the same boundary.
2185        if !self.pause.parked() && !self.interrupt.parked() {
2186            return Ok(false);
2187        }
2188        if self.state.parked {
2189            return Ok(true);
2190        }
2191        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2192        if let Some(block) = &mut self.state.resource_block
2193            && matches!(
2194                block.cause,
2195                crate::run::ResourceBlockCause::Cancelled { .. }
2196            )
2197        {
2198            block.cause = crate::run::ResourceBlockCause::Cancelled {
2199                park: cause.clone(),
2200            };
2201        }
2202        let why = match cause.as_ref().map(ParkCause::label) {
2203            Some(reason) => format!(
2204                "parked after `{}` ({reason}) — resume to carry on from here",
2205                self.state.status.as_str()
2206            ),
2207            None => format!(
2208                "parked after `{}` — resume to carry on from here",
2209                self.state.status.as_str()
2210            ),
2211        };
2212        self.state.event("park", why);
2213        self.state
2214            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2215        self.state.parked = true;
2216        self.state.save()?;
2217        Ok(true)
2218    }
2219
2220    /// Only a new explicit operator approval may retry failed preparation.
2221    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2222        if self.state.prep_failure.is_none() {
2223            return Ok(());
2224        }
2225        let failed_at = self
2226            .state
2227            .events
2228            .iter()
2229            .rev()
2230            .find(|event| event.node == "prep")
2231            .map_or(self.state.updated_at, |event| event.at);
2232        anyhow::ensure!(
2233            approval_at > failed_at,
2234            "preparation retry requires a new explicit release after the failure"
2235        );
2236        self.state.prep_failure = None;
2237        self.state.prep_complete = Some(false);
2238        self.state
2239            .event("prep-retry", "operator approved retry of preparation");
2240        Ok(())
2241    }
2242
2243    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2244    pub fn on_pause(&mut self, pause: Pause) {
2245        self.pause = pause;
2246    }
2247
2248    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2249    /// scheduler asking this one run - and no other - to park so a task
2250    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2251    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2252    /// handle.
2253    pub fn watch_interrupt(&mut self, pause: Pause) {
2254        self.interrupt = pause;
2255    }
2256
2257    /// Abandon this run's own open questions, once `status` has actually
2258    /// settled rather than merely paused.
2259    ///
2260    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2261    /// either back up with the candidates, the review round and the seat
2262    /// sessions already on disk, so a question an implementer asked mid-round
2263    /// may still get a real answer read by a real resume. Only the statuses
2264    /// `resumable` excludes are actually final: the run merged, it reached
2265    /// `Ready` with nothing left to do, it failed outright with no
2266    /// established point to continue from, or every candidate agreed, with
2267    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2268    /// every one of those the seat that asked is gone for good, exactly like
2269    /// the run being deleted under `magi run rm` - so the same cleanup
2270    /// applies, worded for what actually happened instead of "the run was
2271    /// deleted".
2272    ///
2273    /// Best-effort and silent on success: called from every place `status`
2274    /// can land on one of those three, including ones a resumed run revisits,
2275    /// so it must cost nothing when there was nothing open to begin with.
2276    fn settle_questions(&mut self) {
2277        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2278            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2279        }
2280    }
2281
2282    /// The tail of the graph after a trustworthy tally: fold losers, review,
2283    /// gate, merge, and persist.
2284    async fn finish_after_tally(&mut self) -> Result<()> {
2285        // Before the losers are folded: their branches are what the graft
2286        // reads from.
2287        self.graft().await?;
2288        if self.resource_halt()? {
2289            return Ok(());
2290        }
2291        self.fold_losers().await?;
2292        // Before review starts, and again right before the gate: a run's
2293        // review rounds can themselves take long enough for the base to move
2294        // a second time, and the gate is the one node whose "green" gets
2295        // acted on.
2296        self.sync_to_base().await?;
2297        if self.hold_base_sync().await? || self.resource_halt()? {
2298            return Ok(());
2299        }
2300        if self.state.status == RunStatus::AlreadyInBase {
2301            return Ok(());
2302        }
2303        self.review_loop().await?;
2304        if self.resource_halt()? {
2305            return Ok(());
2306        }
2307        self.sync_to_base().await?;
2308        if self.hold_base_sync().await? || self.resource_halt()? {
2309            return Ok(());
2310        }
2311        if self.state.status == RunStatus::AlreadyInBase {
2312            return Ok(());
2313        }
2314        self.gate().await?;
2315        self.merge().await?;
2316        if self.resource_halt()? {
2317            return Ok(());
2318        }
2319        self.state.save()?;
2320        Ok(())
2321    }
2322
2323    // ---------------------------------------------------------------- prep
2324
2325    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2326    /// run with the step and its output in the event; it is never swallowed.
2327    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2328    /// resume starts over instead of finding candidates and missing seats.
2329    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2330        let result = match ensure_setup_config(&mut self.state, repo).await {
2331            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2332            Err(e) => Err(e),
2333        };
2334        if let Err(e) = result {
2335            git::worktree_remove_clean(repo, wt).await.ok();
2336            if in_prep {
2337                self.rollback_prep(repo).await;
2338            }
2339            self.state.status = RunStatus::Blocked;
2340            self.state.event(
2341                "setup",
2342                format!("worktree setup failed in {}: {e:#}", wt.display()),
2343            );
2344            self.state.save()?;
2345            return Err(e);
2346        }
2347        Ok(())
2348    }
2349
2350    /// Undo everything `prep` created, so a resume re-runs it whole.
2351    async fn rollback_prep(&mut self, repo: &Path) {
2352        let root = self.state.worktree_root();
2353        for c in std::mem::take(&mut self.state.candidates) {
2354            if matches!(
2355                git::worktree_remove_clean(repo, &c.worktree).await,
2356                Ok(true)
2357            ) {
2358                git::branch_delete(repo, &c.branch).await.ok();
2359            } else {
2360                self.state.candidates.push(c);
2361            }
2362        }
2363        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2364        for j in 1..=seats {
2365            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2366                .await
2367                .ok();
2368        }
2369        for k in 1..=self.state.config.graph.advisors {
2370            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2371                .await
2372                .ok();
2373        }
2374    }
2375
2376    async fn prep(&mut self) -> Result<()> {
2377        if let Some(reason) = &self.state.prep_failure {
2378            bail!(
2379                "candidate preparation previously failed: {reason}; correct the source before retrying"
2380            );
2381        }
2382        if self
2383            .state
2384            .prep_complete
2385            .unwrap_or(!self.state.candidates.is_empty())
2386        {
2387            return Ok(());
2388        }
2389        let repo = self.state.repo.clone();
2390        // Includes rollback, config, refs, checkout creation and setup. No
2391        // repository mutation may precede the real shared lock acquisition.
2392        let _coordination = self.coordinate(&repo).await?;
2393        let base = self.state.base_commit.clone();
2394        let plan = self.state.sources.plan(&repo, &base).await?;
2395        self.rollback_prep(&repo).await;
2396        if !self.state.candidates.is_empty() {
2397            bail!(
2398                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2399            );
2400        }
2401        self.state.prep_complete = Some(false);
2402        self.state.status = RunStatus::Prep;
2403        self.state.save()?;
2404        let repo = self.state.repo.clone();
2405        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2406        let root = self.state.worktree_root();
2407        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2408
2409        // The hook is the write-time half of the blindness contract; the
2410        // presentation filter in `blind` is the half that cannot be bypassed.
2411        let hooks_dir = self.state.dir().join("hooks");
2412        if self.state.config.blind.commit_msg_hook {
2413            std::fs::create_dir_all(&hooks_dir)
2414                .with_context(|| format!("create {}", hooks_dir.display()))?;
2415            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2416            let path = hooks_dir.join("commit-msg");
2417            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2418            make_executable(&path)?;
2419            // Ref-counted rather than a plain idempotent set: with more than
2420            // one run able to be in flight in the same repository at once
2421            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2422            // true?" check cannot tell "another run of mine still needs
2423            // this" from "nobody does", and the run that happens to finish
2424            // first would disable the hook out from under a sibling still
2425            // relying on it.
2426            git::acquire_worktree_config(&repo).await?;
2427            self.state.enabled_worktree_config = true;
2428        }
2429
2430        // `[worktree] setup` hides its products per worktree, which needs the
2431        // same per-worktree config the hook does. Held until fold, like it.
2432        ensure_setup_config(&mut self.state, &repo).await?;
2433
2434        for (index, (spec, label)) in self
2435            .roles
2436            .implementers
2437            .clone()
2438            .into_iter()
2439            .zip(labels)
2440            .enumerate()
2441        {
2442            let branch = self.state.branch_for(label);
2443            let worktree = root.join(format!("cand-{label}"));
2444            // Persist intent before Git can leave a partially created tree.
2445            // Candidate registration follows setup, so it cannot inventory
2446            // crashes or refusals during creation itself.
2447            self.state.event(
2448                "prep-intent",
2449                serde_json::json!({
2450                    "branch": branch,
2451                    "worktree": worktree,
2452                    "start": start,
2453                    "picks": plan.picks,
2454                })
2455                .to_string(),
2456            );
2457            self.state.save()?;
2458            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2459            if self.state.config.blind.commit_msg_hook {
2460                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2461            }
2462            git::local_exclude(&worktree, "/.magi/").await?;
2463            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2464                // Nothing of this prep may survive: a resume must start over,
2465                // not find half the candidates and skip the rest.
2466                git::worktree_remove_clean(&repo, &worktree).await.ok();
2467                git::branch_delete(&repo, &branch).await.ok();
2468                self.rollback_prep(&repo).await;
2469                self.state.status = RunStatus::Blocked;
2470                self.state
2471                    .event("prep", format!("worktree setup failed: {e:#}"));
2472                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2473                self.state.save()?;
2474                return Err(e);
2475            }
2476            self.state.candidates.push(Candidate {
2477                index,
2478                label,
2479                agent: spec.id.clone(),
2480                branch,
2481                worktree,
2482                summary: String::new(),
2483                stat: String::new(),
2484                files: 0,
2485                commits: 0,
2486                empty: false,
2487                failed: None,
2488                verified_noop: None,
2489                duration_ms: 0,
2490                folded: false,
2491            });
2492            self.state.save()?;
2493            for pick in &plan.picks {
2494                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2495                {
2496                    self.state.status = RunStatus::Blocked;
2497                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2498                    self.state
2499                        .event("prep", format!("cannot apply referenced commit: {e}"));
2500                    self.state.save()?;
2501                    return Err(e);
2502                }
2503            }
2504        }
2505
2506        for j in 1..=self.roles.judges.len() {
2507            let wt = root.join(format!("judge-{j}"));
2508            if !wt.exists() {
2509                git::worktree_add_detached(&repo, &wt, &base).await?;
2510                self.setup_seat_worktree(&repo, &wt, true).await?;
2511            }
2512        }
2513
2514        // Disposable, detached checkouts for the design-deliberation stage's
2515        // advisor seats — the same shape as the judges' above, at the same
2516        // base commit, since advisors also only ever read. Sized off the
2517        // configured count directly rather than a resolved roster: unlike
2518        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2519        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2520        // `prep` has no `ResolvedRoles` field to read a count from here.
2521        if self.state.config.graph.advise {
2522            for k in 1..=self.state.config.graph.advisors {
2523                let wt = root.join(format!("advisor-{k}"));
2524                if !wt.exists() {
2525                    git::worktree_add_detached(&repo, &wt, &base).await?;
2526                    self.setup_seat_worktree(&repo, &wt, true).await?;
2527                }
2528            }
2529        }
2530
2531        // A judge cannot tell it is looking at its own patch — the seats keep
2532        // separate conversations — but a panel that shares agents with the
2533        // field is less independent than it looks, and that is worth saying out
2534        // loud once per run rather than leaving it in the config.
2535        let authors: Vec<&str> = self
2536            .roles
2537            .implementers
2538            .iter()
2539            .map(|a| a.id.as_str())
2540            .collect();
2541        let overlap: Vec<String> = self
2542            .roles
2543            .judges
2544            .iter()
2545            .enumerate()
2546            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2547            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2548            .collect();
2549        if !overlap.is_empty() {
2550            let note = format!(
2551                "{} also authored a candidate; blind, but the panel is less \
2552                 independent than {} distinct agents would be",
2553                overlap.join(", "),
2554                self.roles.judges.len()
2555            );
2556            self.state.event("prep", note);
2557        }
2558
2559        self.state.event(
2560            "prep",
2561            format!(
2562                "{} candidates, {} judges, base {} ({})",
2563                self.state.candidates.len(),
2564                self.roles.judges.len(),
2565                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2566                self.state.base_branch
2567            ),
2568        );
2569        self.state.prep_complete = Some(true);
2570        self.state.status = RunStatus::Implementing;
2571        self.state.save()?;
2572        Ok(())
2573    }
2574
2575    // -------------------------------------------------------------- advise
2576
2577    /// The design-deliberation stage: independent, read-only advisor seats
2578    /// each sketch a design before any implementer touches the repository,
2579    /// and (when at least one produced a usable proposal) a synthesis seat
2580    /// blends them into a brief `implement` carries in every candidate's
2581    /// prompt.
2582    ///
2583    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2584    /// advisors` is the proposal count. Everything here is best-effort and
2585    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2586    /// that cannot reach quota, or a synthesis seat that produced nothing
2587    /// usable all leave `implement` exactly as it was before this stage
2588    /// existed — the task instruction alone — rather than failing the whole
2589    /// competition over an enrichment stage. Every outcome is still recorded
2590    /// as an event, so a run that got nothing from this stage says why.
2591    ///
2592    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2593    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2594    /// resumed run whose stage failed would re-run it, and re-spend the
2595    /// agent calls, on every reentry before `implement`.
2596    ///
2597    /// Also skipped once any candidate shows implementation progress — the
2598    /// exact predicate `implement` itself uses to decide a candidate is no
2599    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2600    /// is not enough: a run created by an older binary that predates this
2601    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2602    /// an already-`Implementing`-or-later run under this build would
2603    /// otherwise walk straight back through `prep` (a no-op once candidates
2604    /// exist) into this node and spawn every advisor seat against worktrees
2605    /// `prep` never recreated — after implementation has already started,
2606    /// which is exactly the invariant this stage exists to guarantee.
2607    async fn advise(&mut self) -> Result<()> {
2608        let implement_untouched = self
2609            .state
2610            .candidates
2611            .iter()
2612            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2613        if !self.state.config.graph.advise || self.state.advise_attempted {
2614            return Ok(());
2615        }
2616        if !implement_untouched {
2617            self.state.event(
2618                "advise",
2619                "skipping the design-deliberation stage: at least one \
2620                 candidate already shows implementation progress, so this \
2621                 run is past the point the stage exists to run before"
2622                    .to_owned(),
2623            );
2624            self.state.advise_attempted = true;
2625            self.state.save()?;
2626            return Ok(());
2627        }
2628        let run_id = self.state.id.clone();
2629        let prompts = self.state.config.prompts.clone();
2630        let instruction = self.state.instruction.clone();
2631        let language = self.state.config.graph.language.clone();
2632        let root = self.state.worktree_root();
2633        let n = self.state.config.graph.advisors;
2634        let where_recorded = self.state.dir().join("run.json");
2635
2636        let seats = match self.state.config.advisors() {
2637            Ok(seats) if !seats.is_empty() => seats,
2638            Ok(_) => {
2639                self.state.event(
2640                    "advise",
2641                    format!(
2642                        "[graph] advisors is 0; skipping the design-deliberation \
2643                         stage and continuing without a synthesis brief (see {})",
2644                        where_recorded.display()
2645                    ),
2646                );
2647                self.state.advise_attempted = true;
2648                self.state.save()?;
2649                return Ok(());
2650            }
2651            Err(e) => {
2652                self.state.event(
2653                    "advise",
2654                    format!(
2655                        "could not resolve advisor seats ({e:#}); continuing \
2656                         without a design-deliberation brief (see {})",
2657                        where_recorded.display()
2658                    ),
2659                );
2660                self.state.advise_attempted = true;
2661                self.state.save()?;
2662                return Ok(());
2663            }
2664        };
2665
2666        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2667        let artifacts = agent::artifacts_dir(&self.state.dir());
2668        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2669
2670        let mut jobs = Vec::new();
2671        for (i, spec) in seats.iter().cloned().enumerate() {
2672            let seat_key = format!("advisor-{}", i + 1);
2673            let seat = self.seat(&seat_key, &spec.id);
2674            jobs.push(SeatJob {
2675                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2676                spec,
2677                seat,
2678                cwd: worktrees[i % worktrees.len()].clone(),
2679                timeout,
2680                allow_write: false,
2681                sessions: false,
2682                artifacts: artifacts.clone(),
2683                stem: seat_key,
2684                handover: None,
2685            });
2686        }
2687
2688        self.state.event(
2689            "advise",
2690            format!(
2691                "{} advisor seat(s) sketching a design in parallel",
2692                jobs.len()
2693            ),
2694        );
2695        let mut quota_losses = Vec::new();
2696        let cache = self.state.config.cache_dir();
2697        let ctx = WaveCtx {
2698            carry_seats: false,
2699            pauses: Some((&self.pause, &self.interrupt)),
2700            run: &run_id,
2701            node: "advise",
2702            prompts: &prompts,
2703            cache: cache.as_deref(),
2704            round: None,
2705        };
2706        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2707        let results = ask_json_wave::<Proposal>(
2708            jobs,
2709            Arc::clone(&self.sem),
2710            self.state.config.graph.retries,
2711            &advisor_roster,
2712            &ctx,
2713            &mut quota_losses,
2714            &mut self.state,
2715            &|p: &Proposal| p.validate(),
2716        )
2717        .await;
2718        if self.state.resource_block.is_some() {
2719            return self.stop_for_resource();
2720        }
2721        self.state.quota.extend(quota_losses);
2722
2723        let mut records = Vec::with_capacity(results.len());
2724        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2725            let agent_id = seat.agent.clone();
2726            self.state.seats.insert(seat.key.clone(), seat);
2727            match res {
2728                Ok((proposal, out)) => {
2729                    self.state
2730                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2731                    records.push(advise::AdvisorRecord::proposed(
2732                        i + 1,
2733                        agent_id,
2734                        proposal,
2735                        out.duration_ms,
2736                    ));
2737                }
2738                Err(e) => {
2739                    self.state.event(
2740                        "advise",
2741                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2742                    );
2743                    records.push(advise::AdvisorRecord::failed(
2744                        i + 1,
2745                        agent_id,
2746                        e.to_string(),
2747                    ));
2748                }
2749            }
2750        }
2751
2752        let mut advice = advise::Advice {
2753            records,
2754            synthesis: None,
2755        };
2756        if advice.proposals().is_empty() {
2757            self.state.event(
2758                "advise",
2759                "no advisor produced a usable proposal; continuing without a \
2760                 synthesis brief"
2761                    .to_owned(),
2762            );
2763        } else {
2764            match self
2765                .synthesize_brief(
2766                    &advice,
2767                    &instruction,
2768                    &language,
2769                    &worktrees[0],
2770                    &artifacts,
2771                    &run_id,
2772                    &prompts,
2773                    cache.as_deref(),
2774                )
2775                .await
2776            {
2777                Ok(Some(text)) => {
2778                    self.state.event(
2779                        "advise",
2780                        "synthesized a design brief for the implementer".to_owned(),
2781                    );
2782                    advice.synthesis = Some(text);
2783                }
2784                Ok(None) => {
2785                    self.state.event(
2786                        "advise",
2787                        "the synthesis seat produced nothing usable; continuing \
2788                         without a design brief"
2789                            .to_owned(),
2790                    );
2791                }
2792                Err(e) => {
2793                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2794                        return Err(e);
2795                    }
2796                    self.state.event(
2797                        "advise",
2798                        format!("could not synthesize a design brief: {e:#}"),
2799                    );
2800                }
2801            }
2802        }
2803        advise::apply_reflection(&mut advice);
2804
2805        self.state.advice = Some(advice);
2806        self.state.advise_attempted = true;
2807        self.state.save()?;
2808        Ok(())
2809    }
2810
2811    /// The synthesis seat: reads every advisor's proposal and blends them
2812    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2813    /// out of [`Runner::advise`] only for readability — it is not called
2814    /// anywhere else.
2815    ///
2816    /// Picked the same way [`crate::talk`]'s standing conversation and
2817    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2818    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2819    /// order (a claude seat, else the first runnable agent in roster order)
2820    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2821    /// in [`crate::config`] for why a dedicated field exists here at all.
2822    #[allow(clippy::too_many_arguments)]
2823    async fn synthesize_brief(
2824        &mut self,
2825        advice: &advise::Advice,
2826        instruction: &str,
2827        language: &str,
2828        cwd: &Path,
2829        artifacts: &Path,
2830        run_id: &str,
2831        prompts: &Prompts,
2832        cache: Option<&Path>,
2833    ) -> Result<Option<String>> {
2834        let chain = agent::pick_chain(
2835            &self.state.config.agents,
2836            self.state.config.roles.synthesizer.as_ref(),
2837            &agent::installed,
2838            "synthesizer",
2839        )?;
2840        let proposals = advice.proposals();
2841        let mut prompt = prompt::with_overlay(
2842            prompt::synthesize_brief(instruction, &proposals, language),
2843            prompts.overlay("advise"),
2844        );
2845        if cache.is_some() {
2846            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2847            // below — see `prompt::build_cache_note`'s doc for why telling a
2848            // read-only seat to build through the shared cache is exactly how
2849            // a sandbox's write refusal gets misread as a defect.
2850            prompt.push('\n');
2851            prompt.push_str(&prompt::build_cache_note("advise", false));
2852        }
2853        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2854        // Each id is tried once, in order; a quota hit, error or unusable
2855        // answer moves to the next. The seat is single-turn (`sessions:
2856        // false`) and the prompt is the whole context, so a fallback agent
2857        // needs nothing carried over.
2858        let mut last = None;
2859        for (n, spec) in chain.iter().enumerate() {
2860            if n > 0 {
2861                self.state
2862                    .event("advise", format!("synthesis falling back to {}", spec.id));
2863            }
2864            let mut seat = self.seat("advise-synthesis", &spec.id);
2865            let wait = self.repository_wait();
2866            let outcome = wait
2867                .scope(agent::invoke(
2868                    spec,
2869                    &mut seat,
2870                    &Invocation {
2871                        coordination: self.state.config.coordination.as_ref(),
2872                        fetched_base: None,
2873                        cwd,
2874                        prompt: &prompt,
2875                        timeout,
2876                        allow_write: false,
2877                        sessions: false,
2878                        artifacts,
2879                        stem: &if n == 0 {
2880                            "advise-synthesis".to_owned()
2881                        } else {
2882                            format!("advise-synthesis-{}", spec.id)
2883                        },
2884                        run: run_id,
2885                        node: "advise",
2886                        cache_dir: None,
2887                        attachments: &[],
2888                        writable: &[],
2889                    },
2890                ))
2891                .await;
2892            if outcome.is_ok() {
2893                self.state.seats.insert(seat.key.clone(), seat);
2894            }
2895            let advance = agent::chain_advances(&outcome);
2896            last = Some(outcome);
2897            if !advance {
2898                break;
2899            }
2900        }
2901        // Exhausted: the last attempt's result is what a single failed seat
2902        // would have produced.
2903        let out = last.expect("a chain holds at least one agent")?;
2904        if !out.usable() {
2905            return Ok(None);
2906        }
2907        let text =
2908            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2909        Ok((!text.trim().is_empty()).then_some(text))
2910    }
2911
2912    // ----------------------------------------------------------- implement
2913
2914    async fn implement(&mut self) -> Result<()> {
2915        self.persist_candidates().await?;
2916        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2917        // agent files with `magi task add` name the run that paid for it. The
2918        // prompt overlay is cloned alongside it because the waves borrow it
2919        // while `self` is mutably borrowed by the node's own bookkeeping.
2920        let run_id = self.state.id.clone();
2921        let prompts = self.state.config.prompts.clone();
2922        let todo: Vec<usize> = self
2923            .state
2924            .candidates
2925            .iter()
2926            .enumerate()
2927            .filter(|(_, c)| {
2928                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2929                    && c.failed.is_none()
2930                    && !c.empty
2931            })
2932            .map(|(i, _)| i)
2933            .collect();
2934        if todo.is_empty() {
2935            return self.after_implement();
2936        }
2937        self.state.status = RunStatus::Implementing;
2938
2939        let language = self.state.config.graph.language.clone();
2940        let github_language = self.state.config.graph.github_language.clone();
2941        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2942        let sessions = self.state.config.graph.sessions;
2943        let artifacts = agent::artifacts_dir(&self.state.dir());
2944        // The design-deliberation stage's blended brief, when `advise` found
2945        // one — carried into every implementer's prompt the same way
2946        // regardless of which candidate it is.
2947        let brief = self
2948            .state
2949            .advice
2950            .as_ref()
2951            .and_then(|a| a.synthesis.as_deref())
2952            .map(str::to_owned);
2953        let attachments = self.state.attachments.clone();
2954
2955        let mut jobs = Vec::new();
2956        for &i in &todo {
2957            let (index, label, worktree) = {
2958                let c = &self.state.candidates[i];
2959                (c.index, c.label, c.worktree.clone())
2960            };
2961            let spec = self.roles.implementers[index].clone();
2962            let seat_key = format!("impl-{label}");
2963            let seat = self.seat(&seat_key, &spec.id);
2964            let instruction = seeded_instruction(&self.state);
2965            jobs.push(SeatJob {
2966                spec,
2967                seat,
2968                prompt: prompt::implement_in(
2969                    &instruction,
2970                    &worktree.to_string_lossy(),
2971                    &language,
2972                    &github_language,
2973                    brief.as_deref(),
2974                    &attachments,
2975                ),
2976                cwd: worktree,
2977                timeout,
2978                allow_write: true,
2979                sessions,
2980                artifacts: artifacts.clone(),
2981                stem: format!("impl-{label}"),
2982                handover: None,
2983            });
2984        }
2985
2986        self.state.event(
2987            "implement",
2988            format!("{} candidates in parallel", jobs.len()),
2989        );
2990        // Kept so a seat whose CLI hung up can be asked again from the same
2991        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2992        // can update a seat's own entry once a fallback agent takes it over —
2993        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2994        // whichever agent actually answered, not the one that quota'd out.
2995        let mut sent = jobs.clone();
2996        let cache = self.state.config.cache_dir();
2997        let ctx = WaveCtx {
2998            carry_seats: false,
2999            pauses: Some((&self.pause, &self.interrupt)),
3000            run: &run_id,
3001            node: "implement",
3002            prompts: &prompts,
3003            cache: cache.as_deref(),
3004            round: None,
3005        };
3006        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3007        if !wave_blocked(&results) {
3008            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3009                .await;
3010            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3011                .await?;
3012            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3013                .await;
3014        }
3015        // Do not finalize failed candidates when the entire wave needs machine
3016        // recovery: leaving them pending resumes this node with its work intact.
3017        if !results.is_empty()
3018            && results
3019                .iter()
3020                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3021        {
3022            let losses: Vec<_> = results
3023                .iter()
3024                .map(|(_, seat, out)| {
3025                    let AgentOutcome::Quota(o) = out else {
3026                        unreachable!()
3027                    };
3028                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3029                })
3030                .collect();
3031            self.state.quota.extend(losses.clone());
3032            for (_, seat, _) in results {
3033                self.state.seats.insert(seat.key.clone(), seat);
3034            }
3035            for &i in &todo {
3036                let label = self.state.candidates[i].label;
3037                self.state.interrupted_implementations.insert(label);
3038                self.state
3039                    .candidate_preservation
3040                    .insert(label, "machine outage awaiting persistence".into());
3041            }
3042            // Preserve partial edits too, while keeping every interrupted seat
3043            // pending even when its tree already contains an inherited patch.
3044            block_machine_outage(&mut self.state, "implement", &losses);
3045            self.state.save()?;
3046            self.persist_candidates().await?;
3047            for &i in &todo {
3048                self.state
3049                    .interrupted_implementations
3050                    .insert(self.state.candidates[i].label);
3051            }
3052            return self.stop_for_resource();
3053        }
3054        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3055
3056        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3057            && results
3058                .iter()
3059                .any(|(_, _, out)| FailClass::of(out).is_some());
3060        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3061            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3062                // Executed turns interrupted by disk pressure already carry
3063                // a preservation barrier from wave. Admission-only waits do
3064                // not. Keep the interrupted seat's session for recovery too.
3065                if self
3066                    .state
3067                    .candidate_preservation
3068                    .contains_key(&self.state.candidates[i].label)
3069                {
3070                    self.state.seats.insert(seat.key.clone(), seat);
3071                }
3072                continue;
3073            }
3074            let seat_key = seat.key.clone();
3075            // A quota fallback (`resume_seat_handovers`) may have handed this
3076            // seat to a different agent than the one `prep` recorded on the
3077            // candidate; the stats tables and any later fixer-defaults-to-
3078            // winner's-author lookup must credit whoever actually answered —
3079            // unless every fallback also quota'd out, in which case nobody
3080            // actually answered and crediting the last agent tried would
3081            // erase every earlier agent's own quota loss from the stats
3082            // tables instead of just this one seat's.
3083            let agent = seat.agent.clone();
3084            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3085            self.state.seats.insert(seat.key.clone(), seat);
3086            let label = self.state.candidates[i].label;
3087            self.state.interrupted_implementations.remove(&label);
3088
3089            let (summary, duration, failed, verified_claim) = match out {
3090                AgentOutcome::Ok(o) => {
3091                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3092                    let failed = (!o.usable()).then(|| {
3093                        if o.timed_out {
3094                            "agent timed out".to_owned()
3095                        } else {
3096                            format!(
3097                                "agent exited with {}",
3098                                crate::notices::exit_status(o.exit_code)
3099                            )
3100                        }
3101                    });
3102                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3103                    (text, o.duration_ms, failed, verified_claim)
3104                }
3105                // Left un-resumed by `resume_undelivered` (a dirty tree
3106                // already rescues the work, or there was no session left to
3107                // resume into) — reported like the ordinary failure it is,
3108                // never as if `o.text` (the CLI's raw error JSON) were an
3109                // answer.
3110                AgentOutcome::Dropped(o) => {
3111                    let why = o
3112                        .dropped
3113                        .as_ref()
3114                        .map(|d| d.why.as_str())
3115                        .unwrap_or("the CLI ended the stream without delivering its answer");
3116                    (
3117                        String::new(),
3118                        o.duration_ms,
3119                        Some(format!("the CLI dropped the stream ({why})")),
3120                        None,
3121                    )
3122                }
3123                AgentOutcome::Quota(o) => {
3124                    self.state.quota.push(QuotaLoss::from_report(
3125                        seat_key,
3126                        "implement",
3127                        o.quota.as_ref(),
3128                    ));
3129                    (
3130                        String::new(),
3131                        o.duration_ms,
3132                        Some("rate limited (quota); produced no change".to_owned()),
3133                        None,
3134                    )
3135                }
3136                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3137                    (String::new(), 0, Some(e), None)
3138                }
3139            };
3140
3141            // Save every completed turn before any fallible Git operation. A failed
3142            // rescue must not erase the reply or prevent collection of other seats.
3143            let c = &mut self.state.candidates[i];
3144            if !exhausted_the_fallback_chain {
3145                c.agent = agent;
3146            }
3147            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3148            c.duration_ms = duration;
3149            c.failed = failed;
3150            c.verified_noop = verified_claim;
3151            self.state
3152                .candidate_preservation
3153                .insert(label, "awaiting persistence".to_owned());
3154        }
3155        self.state.save()?;
3156        self.persist_candidates().await?;
3157
3158        if resource_blocked {
3159            return self.stop_for_resource();
3160        }
3161        if selected_failed {
3162            self.state.status = RunStatus::Blocked;
3163            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3164            self.state.save()?;
3165            bail!(
3166                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3167            );
3168        }
3169        self.after_implement()
3170    }
3171
3172    /// Retry only persistence after a completed implementation turn.
3173    async fn persist_candidates(&mut self) -> Result<()> {
3174        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3175        for label in labels {
3176            let i = self
3177                .state
3178                .candidates
3179                .iter()
3180                .position(|c| c.label == label)
3181                .context("pending candidate is missing")?;
3182            let c = self.state.candidates[i].clone();
3183            let interrupted = self.state.interrupted_implementations.contains(&label);
3184            let result = async {
3185                let wait = crate::repo_lock::WaitPolicy::new(
3186                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3187                    || false,
3188                );
3189                let _coordination =
3190                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3191                // Best effort evidence requires no index writes. A successful snapshot
3192                // never authorizes removal of a tree whose rescue failed.
3193                if let Err(e) = git::snapshot_uncommitted(
3194                    &c.worktree,
3195                    &self
3196                        .state
3197                        .dir()
3198                        .join("artifacts")
3199                        .join(format!("candidate-{label}-recovery")),
3200                )
3201                .await
3202                {
3203                    self.state.event(
3204                        "preserve",
3205                        format!("candidate {label} snapshot incomplete: {e:#}"),
3206                    );
3207                }
3208                let rescued = git::rescue_commit(
3209                    &c.worktree,
3210                    &format!("magi: candidate {label} (uncommitted work)"),
3211                )
3212                .await?;
3213                self.state.note_withheld("implement", &rescued.withheld);
3214                let base = &self.state.base_commit;
3215                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3216                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3217                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3218                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3219                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3220                if c.empty && commits > 0 && !patch.trim().is_empty()
3221                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3222                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3223                        || self.state.tally.is_some())
3224                {
3225                    // Keep the old classification and pending marker so every
3226                    // retry stops too. Retain decisions as recovery evidence;
3227                    // only a supported recovery operation may replace them.
3228                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3229                }
3230                let c = &mut self.state.candidates[i];
3231                c.commits = commits;
3232                c.stat = stat;
3233                c.files = files;
3234                let no_changes = commits == 0 || patch.trim().is_empty();
3235                c.empty = no_changes && !interrupted;
3236                if interrupted && no_changes {
3237                    c.failed = None;
3238                    c.verified_noop = None;
3239                }
3240                if !c.empty {
3241                    c.failed = None;
3242                    c.verified_noop = None;
3243                }
3244                self.state.event(
3245                    "preserve",
3246                    format!("candidate {label}: {files} files, {commits} commits"),
3247                );
3248                Ok::<_, anyhow::Error>(())
3249            }
3250            .await;
3251            match result {
3252                Ok(()) => {
3253                    self.state.candidate_preservation.remove(&label);
3254                    if !interrupted || self.state.candidates[i].files > 0 {
3255                        self.state.interrupted_implementations.remove(&label);
3256                    }
3257                }
3258                Err(e) => {
3259                    let reason = format!("candidate {label} preservation failed: {e:#}");
3260                    self.state.event("preserve", &reason);
3261                    self.state.candidate_preservation.insert(label, reason);
3262                }
3263            }
3264            if self.state.candidate_preservation.is_empty()
3265                && self.state.status == RunStatus::Blocked
3266                && self.state.candidates.iter().all(|c| c.failed.is_none())
3267            {
3268                // Persist the recovered phase with the cleared pending marker;
3269                // a crash between saves must not turn this into a selected-seat
3270                // failure hold on the next resume.
3271                self.state.status = RunStatus::Implementing;
3272                self.state.event(
3273                    "preserve",
3274                    "candidate persistence recovered; implementation can continue",
3275                );
3276            }
3277            self.state.save()?;
3278        }
3279        if !self.state.candidate_preservation.is_empty() {
3280            self.state.status = RunStatus::Blocked;
3281            self.state.save()?;
3282            bail!(
3283                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3284            );
3285        }
3286        Ok(())
3287    }
3288
3289    /// Ask again, once, for work a CLI did and then failed to hand over.
3290    ///
3291    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3292    /// status with an empty response and a usage report showing output tokens,
3293    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3294    /// seven minutes and 14,267 output tokens that arrived as an empty
3295    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3296    ///
3297    /// Two conditions, and both matter:
3298    ///
3299    /// - **Only when the tree is untouched.** Often the agent has already
3300    ///   written its files and only the closing message was lost; the rescue
3301    ///   commit below picks that up and there is nothing to ask for. Re-asking
3302    ///   then would pay for a second implementation of work already on disk.
3303    /// - **Once.** A CLI that drops one stream can drop the next, and this
3304    ///   node is the most expensive in the graph.
3305    ///
3306    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3307    /// dropped reply still carried its `conversation_id`, so the seat is asked
3308    /// to finish what it was doing rather than sent the whole task again. It
3309    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3310    /// node's - for the same reason a re-ranked judge does: restating finished
3311    /// work is not the work.
3312    ///
3313    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3314    /// same way until it resets, while an abandoned conversation is still
3315    /// there to be picked up.
3316    async fn resume_undelivered(
3317        &mut self,
3318        results: &mut [(usize, SeatState, AgentOutcome)],
3319        sent: &[SeatJob],
3320        prompts: &Prompts,
3321        run_id: &str,
3322    ) {
3323        for (wi, seat, out) in results.iter_mut() {
3324            let Some(dropped) = (match &*out {
3325                AgentOutcome::Dropped(o) => o.dropped.clone(),
3326                _ => None,
3327            }) else {
3328                continue;
3329            };
3330            let Some(job) = sent.get(*wi) else { continue };
3331            // Already on disk? Then only the closing message was lost.
3332            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3333                self.state.event(
3334                    "implement",
3335                    format!(
3336                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3337                         work is in the tree",
3338                        seat.key, dropped.output_tokens, dropped.why
3339                    ),
3340                );
3341                continue;
3342            }
3343            // The re-ask only makes sense as a resume: `resume_after_drop`
3344            // says nothing about the task, trusting the seat to still hold it.
3345            // Without a session to resume — sessions disabled, or this CLI's
3346            // drop shape happened not to carry a session id — that prompt
3347            // would open a brand-new conversation with no context at all,
3348            // which is worse than leaving this as the ordinary failure it
3349            // already is.
3350            if !has_context(&job.spec, seat, job.sessions) {
3351                self.state.event(
3352                    "implement",
3353                    format!(
3354                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3355                         is no session left to resume",
3356                        seat.key, dropped.output_tokens, dropped.why
3357                    ),
3358                );
3359                continue;
3360            }
3361            self.state.event(
3362                "implement",
3363                format!(
3364                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3365                     conversation",
3366                    seat.key, dropped.output_tokens, dropped.why
3367                ),
3368            );
3369            let mut retry = job.clone();
3370            retry.seat = seat.clone();
3371            retry.prompt = prompt::resume_after_drop(&dropped.why);
3372            retry.timeout = retry_budget(job.timeout, true);
3373            retry.stem = format!("{}-resume", job.stem);
3374            let cache = self.state.config.cache_dir();
3375            let ctx = WaveCtx {
3376                carry_seats: false,
3377                pauses: Some((&self.pause, &self.interrupt)),
3378                run: run_id,
3379                node: "implement",
3380                prompts,
3381                cache: cache.as_deref(),
3382                round: None,
3383            };
3384            let (resumed_seat, resumed) =
3385                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3386            *seat = resumed_seat;
3387            *out = resumed;
3388        }
3389    }
3390
3391    /// Fall an implement seat through to the next untried agent in the
3392    /// implementer roster when it lost to quota — or, since the handover was
3393    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3394    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3395    /// quota path itself being unchanged — instead of leaving the
3396    /// seat's loss final the moment one agent's account runs dry.
3397    ///
3398    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3399    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3400    /// truncates to the single slot rotation picked, so a solo task whose one
3401    /// implementer hits quota mid-run used to have nothing else to try. This
3402    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3403    /// unrotated roster — which is the only place the *other* candidates in
3404    /// the machine's roster still exist once `implementers` has been cut down
3405    /// to size.
3406    ///
3407    /// Walks forward from just past the seat's own original position in the
3408    /// roster, never wrapping back to the front: a later candidate slot (say
3409    /// `beta`, the roster's second entry) must fall through to the *next*
3410    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3411    /// almost certainly a different candidate's own agent already — and once
3412    /// the roster's tail is exhausted there is nothing left to fall through
3413    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3414    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3415    /// let this retry that id forever. The loop keeps falling through until
3416    /// an attempt lands something other than `Quota` or the roster's tail
3417    /// runs out of untried ids, at which point the seat is left exactly as
3418    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3419    /// `QuotaLoss` recorded, the candidate failed/empty.
3420    ///
3421    /// `sent` is taken mutably and updated with the fallback agent's spec:
3422    /// `resume_unconfirmed_commands`, which runs after this and also reads
3423    /// `sent`, must see whichever agent actually ended up answering the seat
3424    /// — reading the stale, original spec there would check session
3425    /// eligibility against the wrong CLI and could hand a fallback agent's
3426    /// session id to the agent that just lost the seat to quota.
3427    ///
3428    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3429    /// — `self.seat` only reuses state when the agent id is unchanged, so
3430    /// handing it a different id already gets this for free. Reusing the old
3431    /// seat would resume a different CLI's session as if it were a
3432    /// continuation of this one.
3433    ///
3434    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3435    /// a quota loss cuts an agent off mid-turn, so anything already in the
3436    /// tree is unfinished work, not a completed candidate a re-ask would pay
3437    /// for twice. A dirty tree is rescued into a commit first (the same
3438    /// neutral-identity rescue `implement`'s own outcome loop gives every
3439    /// candidate) so the next agent starts clean.
3440    ///
3441    /// The new agent gets the implementer's full prompt and full
3442    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3443    /// it has no session and no context, and is implementing the task from
3444    /// nothing, unlike a resumed drop which is only restating work already
3445    /// done.
3446    ///
3447    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3448    /// `implement` event, never into `self.state.quota` — that is what
3449    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3450    /// go unspent, and a seat that ultimately recovered on its second or
3451    /// third agent is not the stalled panel that check exists to catch. Only
3452    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3453    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3454    /// outcome loop already has — this helper never pushes to it itself.
3455    async fn resume_seat_handovers(
3456        &mut self,
3457        results: &mut [(usize, SeatState, AgentOutcome)],
3458        sent: &mut [SeatJob],
3459        prompts: &Prompts,
3460        run_id: &str,
3461    ) -> Result<()> {
3462        let instruction = seeded_instruction(&self.state);
3463        let language = self.state.config.graph.language.clone();
3464        let github_language = self.state.config.graph.github_language.clone();
3465        let brief = self
3466            .state
3467            .advice
3468            .as_ref()
3469            .and_then(|a| a.synthesis.as_deref())
3470            .map(str::to_owned);
3471        let attachments = self.state.attachments.clone();
3472        for (wi, seat, out) in results.iter_mut() {
3473            // Who holds the other candidate seats of this wave right now
3474            // (earlier handovers already written back to `sent`).
3475            let others: BTreeSet<String> = sent
3476                .iter()
3477                .enumerate()
3478                .filter(|(j, _)| j != wi)
3479                .map(|(_, j)| j.spec.id.clone())
3480                .collect();
3481            let Some(job) = sent.get_mut(*wi) else {
3482                continue;
3483            };
3484            // Where the seat's own original agent sits in the roster — the
3485            // fallback walk starts just past here, never at the front, so a
3486            // later candidate slot's quota loss does not fall back onto an
3487            // earlier slot's own agent.
3488            let start = self
3489                .roles
3490                .implementer_roster
3491                .iter()
3492                .position(|s| s.id == job.spec.id)
3493                .unwrap_or(0);
3494            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3495            let mut fallback_attempt = 0usize;
3496            let mut prev: Option<FailClass> = None;
3497            while let Some(cur) = FailClass::of(&*out) {
3498                if !should_hand_over(prev.as_ref(), &cur) {
3499                    break;
3500                }
3501                let Some(next) =
3502                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3503                        .cloned()
3504                else {
3505                    break;
3506                };
3507                tried.insert(next.id.clone());
3508                fallback_attempt += 1;
3509
3510                let coordination = self.coordinate(&job.cwd).await?;
3511                if let Ok(r) = git::rescue_commit(
3512                    &job.cwd,
3513                    &format!(
3514                        "magi: candidate {} (uncommitted work before {} fallback)",
3515                        seat.key,
3516                        if cur == FailClass::Quota {
3517                            "quota"
3518                        } else {
3519                            "handover"
3520                        }
3521                    ),
3522                )
3523                .await
3524                {
3525                    self.state.note_withheld("implement", &r.withheld);
3526                }
3527                drop(coordination);
3528
3529                record_handover(
3530                    &mut self.state,
3531                    "implement",
3532                    &seat.key,
3533                    &seat.agent,
3534                    &next.id,
3535                    &cur,
3536                    &fail_reason(&*out),
3537                );
3538                prev = Some(cur.clone());
3539
3540                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3541                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3542                // Kept in sync on `sent` itself, not just the local retry: a
3543                // later helper (`resume_unconfirmed_commands`) reads `sent`
3544                // after this one returns and must see whichever agent is now
3545                // occupying the seat, not the one that just quota'd out —
3546                // otherwise it would judge session/continuation eligibility
3547                // by the wrong CLI and could resend a fallback's session id
3548                // to the agent that lost it the seat in the first place.
3549                job.spec = next.clone();
3550                let mut retry = job.clone();
3551                retry.seat = new_seat;
3552                retry.prompt = prompt::implement_in(
3553                    &instruction,
3554                    &job.cwd.to_string_lossy(),
3555                    &language,
3556                    &github_language,
3557                    brief.as_deref(),
3558                    &attachments,
3559                );
3560                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3561                let cache = self.state.config.cache_dir();
3562                let ctx = WaveCtx {
3563                    carry_seats: false,
3564                    pauses: Some((&self.pause, &self.interrupt)),
3565                    run: run_id,
3566                    node: "implement",
3567                    prompts,
3568                    cache: cache.as_deref(),
3569                    round: None,
3570                };
3571                let (fallback_seat, fallback_out) = run_one(
3572                    retry,
3573                    Arc::clone(&self.sem),
3574                    &ctx,
3575                    &mut self.state,
3576                    fallback_attempt,
3577                )
3578                .await;
3579                *seat = fallback_seat;
3580                *out = fallback_out;
3581            }
3582        }
3583        Ok(())
3584    }
3585
3586    /// Ask an implement seat's own CLI to confirm what it started, once, when
3587    /// its reply reported a command whose completion status it never
3588    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3589    /// that does and does not mean.
3590    ///
3591    /// The completion contract this task asks for, extended to `implement`
3592    /// with the same signal `continue_fix_report` reads for the fixer,
3593    /// rather than a keyword search over the reply or a hard requirement on
3594    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3595    /// a candidate's CLI turn ended cleanly while a test run it had started
3596    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3597    /// named in it at all is untouched by this: `commands` is empty, so
3598    /// there is nothing to be unconfirmed.
3599    ///
3600    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3601    /// this is not about recovering edits that might already be on disk, it
3602    /// is about a result the seat itself never vouched for, which resuming
3603    /// asks for regardless of what the tree already holds. Bounded to one
3604    /// attempt for the same reason `resume_undelivered` is — this is the
3605    /// most expensive node in the graph — and a seat that still cannot
3606    /// confirm on that attempt is left as whatever its (possibly still
3607    /// unconfirmed) reply says; this does not invent a new "failed" reason
3608    /// for a candidate that otherwise produced a real, committed change.
3609    async fn resume_unconfirmed_commands(
3610        &mut self,
3611        results: &mut [(usize, SeatState, AgentOutcome)],
3612        sent: &[SeatJob],
3613        prompts: &Prompts,
3614        run_id: &str,
3615    ) {
3616        for (wi, seat, out) in results.iter_mut() {
3617            let AgentOutcome::Ok(o) = &*out else {
3618                continue;
3619            };
3620            if !has_unconfirmed_command(&o.commands) {
3621                continue;
3622            }
3623            let Some(job) = sent.get(*wi) else { continue };
3624            if !has_context(&job.spec, seat, job.sessions) {
3625                self.state.event(
3626                    "implement",
3627                    format!(
3628                        "{}: the reply named a command whose own CLI never confirmed the exit \
3629                         status of, but there is no session left to resume",
3630                        seat.key
3631                    ),
3632                );
3633                continue;
3634            }
3635            self.state.event(
3636                "implement",
3637                format!(
3638                    "{}: the reply named a command whose own CLI never confirmed the exit \
3639                     status of; resuming the conversation",
3640                    seat.key
3641                ),
3642            );
3643            let mut retry = job.clone();
3644            retry.seat = seat.clone();
3645            retry.prompt = prompt::resume_incomplete(
3646                "a command in your last reply had no confirmed exit status",
3647            );
3648            retry.timeout = retry_budget(job.timeout, true);
3649            retry.stem = format!("{}-confirm", job.stem);
3650            let cache = self.state.config.cache_dir();
3651            let ctx = WaveCtx {
3652                carry_seats: false,
3653                pauses: Some((&self.pause, &self.interrupt)),
3654                run: run_id,
3655                node: "implement",
3656                prompts,
3657                cache: cache.as_deref(),
3658                round: None,
3659            };
3660            let (resumed_seat, resumed) =
3661                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3662            *seat = resumed_seat;
3663            *out = resumed;
3664        }
3665    }
3666
3667    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3668    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3669    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3670    /// that motivated this.
3671    ///
3672    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3673    /// own nudge loop already covers for judge/review/vote seats, and not a
3674    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3675    /// implement seats: here the CLI turn genuinely finished while the node's
3676    /// own work — the fixer's account of what it did — had not. Gated purely
3677    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3678    /// reply, never on any wording in it, so a fixer whose valid, first-try
3679    /// `FixReport` happens to mention having waited on a background test is
3680    /// never resumed — the `Ok(report)` branch at the call site returns
3681    /// before this is ever invoked.
3682    ///
3683    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3684    /// attempt ([`retry_budget`]), nothing attempted once the session is
3685    /// gone, and a quota hit ends the loop immediately rather than retrying a
3686    /// rate limit that fails the same way again.
3687    async fn continue_fix_report(
3688        &mut self,
3689        mut seat: SeatState,
3690        parse_err: String,
3691        job: &SeatJob,
3692        prompts: &Prompts,
3693        run_id: &str,
3694        round: usize,
3695    ) -> (
3696        SeatState,
3697        Option<FixReport>,
3698        Option<String>,
3699        ContinuationRecord,
3700    ) {
3701        let mut last_err = parse_err;
3702        let mut cumulative_wait_ms = 0u64;
3703        let mut attempts = 0usize;
3704        loop {
3705            if !has_context(&job.spec, &seat, job.sessions) {
3706                self.state.event(
3707                    "fix",
3708                    format!(
3709                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3710                         session left to resume into"
3711                    ),
3712                );
3713                let outcome = if attempts == 0 {
3714                    ContinuationOutcome::NoSession
3715                } else {
3716                    ContinuationOutcome::Exhausted
3717                };
3718                return (
3719                    seat,
3720                    None,
3721                    Some(format!("unparsable fix report: {last_err}")),
3722                    ContinuationRecord {
3723                        attempts,
3724                        cumulative_wait_ms,
3725                        outcome,
3726                    },
3727                );
3728            }
3729            if attempts >= MAX_FIX_CONTINUATIONS {
3730                self.state.event(
3731                    "fix",
3732                    format!(
3733                        "round {round}: fixer's reply still had no adoption report after \
3734                         {attempts} continuation(s) ({last_err}); giving up"
3735                    ),
3736                );
3737                return (
3738                    seat,
3739                    None,
3740                    Some(format!(
3741                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3742                    )),
3743                    ContinuationRecord {
3744                        attempts,
3745                        cumulative_wait_ms,
3746                        outcome: ContinuationOutcome::Exhausted,
3747                    },
3748                );
3749            }
3750            attempts += 1;
3751            self.state.event(
3752                "fix",
3753                format!(
3754                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3755                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3756                ),
3757            );
3758            let mut retry = job.clone();
3759            retry.seat = seat.clone();
3760            retry.prompt = prompt::resume_incomplete(&last_err);
3761            retry.timeout = retry_budget(job.timeout, true);
3762            retry.stem = format!("{}-continue{attempts}", job.stem);
3763            let cache = self.state.config.cache_dir();
3764            let ctx = WaveCtx {
3765                carry_seats: false,
3766                pauses: Some((&self.pause, &self.interrupt)),
3767                run: run_id,
3768                node: "fix",
3769                prompts,
3770                cache: cache.as_deref(),
3771                round: Some(round),
3772            };
3773            let (resumed_seat, resumed_out) = run_one(
3774                retry,
3775                Arc::clone(&self.sem),
3776                &ctx,
3777                &mut self.state,
3778                attempts,
3779            )
3780            .await;
3781            seat = resumed_seat;
3782            match resumed_out {
3783                AgentOutcome::Ok(o) => {
3784                    cumulative_wait_ms += o.duration_ms;
3785                    match verdict::extract_json::<FixReport>(&o.text) {
3786                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3787                            self.state.event(
3788                                "fix",
3789                                format!(
3790                                    "round {round}: fixer's adoption report recovered after \
3791                                     {attempts} continuation(s)"
3792                                ),
3793                            );
3794                            return (
3795                                seat,
3796                                Some(report),
3797                                None,
3798                                ContinuationRecord {
3799                                    attempts,
3800                                    cumulative_wait_ms,
3801                                    outcome: ContinuationOutcome::Resumed,
3802                                },
3803                            );
3804                        }
3805                        // The report parsed, but this same reply's own
3806                        // CommandEvidence — the identical record `state.jobs`
3807                        // renders — names a command whose CLI never
3808                        // confirmed an exit status. Read together, that is
3809                        // not a resolved answer: keep nudging rather than
3810                        // accept a report standing next to a command the
3811                        // seat's own CLI cannot vouch for.
3812                        Ok(_) => {
3813                            last_err = "the reply parsed, but it reported a command whose own CLI \
3814                                 never confirmed an exit status"
3815                                .to_owned();
3816                        }
3817                        Err(e) => last_err = e.to_string(),
3818                    }
3819                }
3820                AgentOutcome::Quota(o) => {
3821                    cumulative_wait_ms += o.duration_ms;
3822                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3823                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3824                    self.state.quota.push(loss);
3825                    self.state.event(
3826                        "fix",
3827                        format!(
3828                            "round {round}: continuation rate limited (quota); not retrying now"
3829                        ),
3830                    );
3831                    return (
3832                        seat,
3833                        None,
3834                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3835                        ContinuationRecord {
3836                            attempts,
3837                            cumulative_wait_ms,
3838                            outcome: ContinuationOutcome::QuotaLost,
3839                        },
3840                    );
3841                }
3842                AgentOutcome::Dropped(o) => {
3843                    cumulative_wait_ms += o.duration_ms;
3844                    let why = o
3845                        .dropped
3846                        .as_ref()
3847                        .map(|d| d.why.as_str())
3848                        .unwrap_or("the CLI ended the stream without delivering its answer");
3849                    last_err = format!("the CLI dropped the stream ({why})");
3850                }
3851                AgentOutcome::Failed(e) => last_err = e,
3852                // The continuation was never started; the report stays lost
3853                // and the caller records that, without any handover.
3854                AgentOutcome::ResourceBlocked(e) => {
3855                    return (
3856                        seat,
3857                        None,
3858                        Some(format!("the continuation was not started: {e}")),
3859                        ContinuationRecord {
3860                            attempts: attempts.saturating_sub(1),
3861                            cumulative_wait_ms,
3862                            outcome: ContinuationOutcome::NoSession,
3863                        },
3864                    );
3865                }
3866            }
3867        }
3868    }
3869
3870    fn after_implement(&mut self) -> Result<()> {
3871        if !self.state.candidate_preservation.is_empty() {
3872            bail!("candidate preservation incomplete; judging is unsafe");
3873        }
3874        // Scan every candidate patch once the set is complete.
3875        if self.state.leaks.is_empty() {
3876            let cfg = self.state.config.blind.clone();
3877            let mut leaks = Vec::new();
3878            for c in &self.state.candidates {
3879                let Some(patch) =
3880                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3881                else {
3882                    continue;
3883                };
3884                leaks.extend(blind::scan(
3885                    &format!("candidate {} patch", c.label),
3886                    &patch,
3887                    &cfg.vendor_tokens,
3888                ));
3889            }
3890            if !leaks.is_empty() {
3891                let summary = leaks
3892                    .iter()
3893                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3894                    .collect::<Vec<_>>()
3895                    .join(", ");
3896                match cfg.on_leak {
3897                    LeakPolicy::Fail => {
3898                        self.state.status = RunStatus::Failed;
3899                        self.state
3900                            .event("blind", format!("vendor text in a patch: {summary}"));
3901                        self.state.leaks = leaks;
3902                        self.state.save()?;
3903                        self.settle_questions();
3904                        bail!(
3905                            "blind.on_leak = \"fail\" and vendor text reached a \
3906                             judged patch: {summary}"
3907                        );
3908                    }
3909                    LeakPolicy::Redact => self.state.event(
3910                        "blind",
3911                        format!("redacting vendor text for judging: {summary}"),
3912                    ),
3913                    LeakPolicy::Warn => self.state.event(
3914                        "blind",
3915                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3916                    ),
3917                }
3918                self.state.leaks = leaks;
3919            }
3920        }
3921
3922        if self.state.viable().is_empty() {
3923            if self.state.all_candidates_verified_noop() {
3924                // Every candidate agreed, with evidence the adoption guard
3925                // accepted, that nothing belongs in this worktree. That is
3926                // not the same fact as a candidate that simply failed to
3927                // write anything, and settling it as an ordinary `Failed`
3928                // (see `SCHEMA`'s doc for schema 10) is what let two of
3929                // task 391f's attempts burn a retry each re-discovering the
3930                // same already-landed fix. Terminal either way, so `judge`
3931                // must never run over an empty candidate set — unlike the
3932                // `Failed` branch below this returns `Ok`, not an error:
3933                // nothing here failed.
3934                self.state.status = RunStatus::VerifiedNoop;
3935                self.state.save()?;
3936                self.settle_questions();
3937                return Ok(());
3938            }
3939            self.state.status = RunStatus::Failed;
3940            self.state.save()?;
3941            self.settle_questions();
3942            bail!("no candidate produced a change; nothing to judge");
3943        }
3944        self.state.status = RunStatus::Judging;
3945        self.state.save()?;
3946        Ok(())
3947    }
3948
3949    // --------------------------------------------------------------- judge
3950
3951    async fn judge(&mut self) -> Result<()> {
3952        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3953        // agent files with `magi task add` name the run that paid for it. The
3954        // prompt overlay is cloned alongside it because the waves borrow it
3955        // while `self` is mutably borrowed by the node's own bookkeeping.
3956        let run_id = self.state.id.clone();
3957        let prompts = self.state.config.prompts.clone();
3958        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3959            return Ok(());
3960        }
3961        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3962        if viable.len() == 1 {
3963            // Recorded so this is a one-time event: `judgements` stays empty
3964            // either way, which without this flag is indistinguishable from
3965            // "not yet judged" on the next reentry — and status is left
3966            // untouched, so a later node's conclusion (e.g. `Blocked` after
3967            // the review budget ran out) survives a resume instead of being
3968            // clobbered back to `Judging` by this node running again.
3969            self.state.judge_skipped = true;
3970            self.state.event(
3971                "judge",
3972                format!(
3973                    "only candidate {} produced a change; judging skipped",
3974                    viable[0].label
3975                ),
3976            );
3977            self.state.save()?;
3978            return Ok(());
3979        }
3980        self.state.status = RunStatus::Judging;
3981
3982        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3983        let language = self.state.config.graph.language.clone();
3984        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3985        let sessions = self.state.config.graph.sessions;
3986        let artifacts = agent::artifacts_dir(&self.state.dir());
3987        let root = self.state.worktree_root();
3988        let base_short = short(&self.state.base_commit);
3989
3990        let mut jobs = Vec::new();
3991        let mut orders = Vec::new();
3992        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3993            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3994            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3995            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3996            let seat_key = format!("judge-{}", j + 1);
3997            let seat = self.seat(&seat_key, &spec.id);
3998            jobs.push(SeatJob {
3999                prompt: prompt::judge(
4000                    &self.state.instruction,
4001                    &views,
4002                    self.roles.judges.len(),
4003                    &base_short,
4004                    &language,
4005                ),
4006                spec,
4007                seat,
4008                cwd: root.join(format!("judge-{}", j + 1)),
4009                timeout,
4010                allow_write: false,
4011                sessions,
4012                artifacts: artifacts.clone(),
4013                stem: format!("judge-{}", j + 1),
4014                handover: None,
4015            });
4016        }
4017
4018        self.state.event(
4019            "judge",
4020            format!(
4021                "{} judges ranking {} candidates blind",
4022                jobs.len(),
4023                viable.len()
4024            ),
4025        );
4026        let labels_for_check = labels.clone();
4027        let mut quota_losses = Vec::new();
4028        let cache = self.state.config.cache_dir();
4029        let ctx = WaveCtx {
4030            carry_seats: false,
4031            pauses: Some((&self.pause, &self.interrupt)),
4032            run: &run_id,
4033            node: "judge",
4034            prompts: &prompts,
4035            cache: cache.as_deref(),
4036            round: None,
4037        };
4038        let results = ask_json_wave::<Ranking>(
4039            jobs,
4040            Arc::clone(&self.sem),
4041            self.state.config.graph.retries,
4042            &self.roles.judge_roster,
4043            &ctx,
4044            &mut quota_losses,
4045            &mut self.state,
4046            &move |r: &Ranking| r.validate(&labels_for_check),
4047        )
4048        .await;
4049        if self.state.resource_block.is_some() {
4050            return self.stop_for_resource();
4051        }
4052        self.state.quota.extend(quota_losses);
4053
4054        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4055            let agent_id = seat.agent.clone();
4056            self.state.seats.insert(seat.key.clone(), seat);
4057            let mut record = Judgement {
4058                judge: j + 1,
4059                seat: format!("judge-{}", j + 1),
4060                agent: agent_id,
4061                ranking: Vec::new(),
4062                reasons: BTreeMap::new(),
4063                confidence: None,
4064                strengths: BTreeMap::new(),
4065                order: orders[j].clone(),
4066                failed: None,
4067                duration_ms: 0,
4068            };
4069            match res {
4070                Ok((ranking, out)) => {
4071                    record.ranking = ranking.normalized();
4072                    record.reasons = ranking.reasons;
4073                    record.strengths = ranking.strengths;
4074                    record.confidence = ranking.confidence;
4075                    record.duration_ms = out.duration_ms;
4076                    self.state.event(
4077                        "judge",
4078                        format!(
4079                            "judge {} ranked {}",
4080                            j + 1,
4081                            record.ranking.iter().collect::<String>()
4082                        ),
4083                    );
4084                }
4085                Err(e) => {
4086                    record.failed = Some(e.to_string());
4087                    self.state
4088                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4089                }
4090            }
4091            self.state.judgements.push(record);
4092            self.state.save()?;
4093        }
4094        Ok(())
4095    }
4096
4097    // ---------------------------------------------------------- deliberate
4098
4099    async fn deliberate(&mut self) -> Result<()> {
4100        if self.hold_base_sync().await? {
4101            return Ok(());
4102        }
4103        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4104        // agent files with `magi task add` name the run that paid for it. The
4105        // prompt overlay is cloned alongside it because the waves borrow it
4106        // while `self` is mutably borrowed by the node's own bookkeeping.
4107        let run_id = self.state.id.clone();
4108        let prompts = self.state.config.prompts.clone();
4109        let rounds = self.state.config.graph.deliberate_rounds;
4110        // Resume unfinished rounds after an outage, but never reopen a tally.
4111        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4112            return Ok(());
4113        }
4114        let tops: Vec<char> = self
4115            .state
4116            .judgements
4117            .iter()
4118            .filter_map(|j| j.ranking.first().copied())
4119            .collect();
4120        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4121            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4122                self.state.event(
4123                    "deliberate",
4124                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4125                );
4126            }
4127            self.state.status = RunStatus::Voting;
4128            self.state.save()?;
4129            return Ok(());
4130        }
4131
4132        self.state.status = RunStatus::Deliberating;
4133        self.state.event(
4134            "deliberate",
4135            format!(
4136                "split: first choices were {} — opening {rounds} round(s)",
4137                tops.iter().collect::<String>()
4138            ),
4139        );
4140
4141        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4142        let language = self.state.config.graph.language.clone();
4143        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4144        let sessions = self.state.config.graph.sessions;
4145        let artifacts = agent::artifacts_dir(&self.state.dir());
4146        let root = self.state.worktree_root();
4147        let base_short = short(&self.state.base_commit);
4148
4149        // Judges argue in sequence so that a turn can answer the one before it;
4150        // that is the difference between deliberation and three parallel
4151        // monologues.
4152        for round in self.state.deliberation.len() + 1..=rounds {
4153            let mut turns: Vec<DeliberationTurn> = Vec::new();
4154            let mut round_losses = Vec::new();
4155            let mut asked = 0;
4156            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4157                if self.state.judgements[j].failed.is_some() {
4158                    continue;
4159                }
4160                let seat_key = format!("judge-{}", j + 1);
4161                let spec = self.occupant(&seat_key, spec);
4162                let mut seat = self.seat(&seat_key, &spec.id);
4163                let transcript = self.transcript(&turns, j);
4164                let build = |context: Option<&str>| {
4165                    prompt::deliberate(
4166                        &self.state.instruction,
4167                        context,
4168                        &transcript,
4169                        round,
4170                        rounds,
4171                        &language,
4172                    )
4173                };
4174                let block = self.candidate_block(&viable, &base_short);
4175                let full = build(Some(&block));
4176                let text = if has_context(&spec, &seat, sessions) {
4177                    build(None)
4178                } else {
4179                    full.clone()
4180                };
4181                let job = SeatJob {
4182                    spec,
4183                    seat: seat.clone(),
4184                    prompt: text,
4185                    cwd: root.join(format!("judge-{}", j + 1)),
4186                    timeout,
4187                    allow_write: false,
4188                    sessions,
4189                    artifacts: artifacts.clone(),
4190                    stem: format!("delib-{round}-judge-{}", j + 1),
4191                    handover: Some(full),
4192                };
4193                let cache = self.state.config.cache_dir();
4194                let ctx = WaveCtx {
4195                    carry_seats: false,
4196                    pauses: Some((&self.pause, &self.interrupt)),
4197                    run: &run_id,
4198                    node: "deliberate",
4199                    prompts: &prompts,
4200                    cache: cache.as_deref(),
4201                    round: None,
4202                };
4203                // A turn is never nudged (`retries` 0): a failed seat is
4204                // handed to the next roster agent, which gets the full
4205                // context. An empty answer is a turn, not a failure.
4206                let mut losses = Vec::new();
4207                asked += 1;
4208                let mut results = ask_wave_with::<String>(
4209                    vec![job],
4210                    Arc::clone(&self.sem),
4211                    0,
4212                    &self.roles.judge_roster,
4213                    &ctx,
4214                    &mut losses,
4215                    &mut self.state,
4216                    &|text: &str| {
4217                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4218                    },
4219                )
4220                .await;
4221                if self.state.resource_block.is_some() {
4222                    return self.stop_for_resource();
4223                }
4224                round_losses.extend(losses.iter().cloned());
4225                self.state.quota.extend(losses);
4226                let (updated, res, _) = results.pop().expect("one job in, one result out");
4227                seat = updated;
4228                let agent_id = seat.agent.clone();
4229                self.state.seats.insert(seat.key.clone(), seat);
4230                let body = match res {
4231                    Ok((body, _)) => body,
4232                    // Skip the seat; a CLI's raw error JSON is never read as
4233                    // this judge's position.
4234                    Err(e) => {
4235                        self.state
4236                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4237                        continue;
4238                    }
4239                };
4240                let tentative = verdict::extract_json::<Position>(&body)
4241                    .ok()
4242                    .and_then(|p| p.tentative)
4243                    .and_then(|s| s.trim().chars().next())
4244                    .map(|c| c.to_ascii_uppercase());
4245                self.state.event(
4246                    "deliberate",
4247                    format!(
4248                        "round {round}: judge {} now favours {}",
4249                        j + 1,
4250                        tentative.map_or("—".to_owned(), |c| c.to_string())
4251                    ),
4252                );
4253                turns.push(DeliberationTurn {
4254                    judge: j + 1,
4255                    agent: agent_id,
4256                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4257                    tentative,
4258                });
4259            }
4260            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4261                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4262                return self.stop_for_resource();
4263            }
4264            self.state
4265                .deliberation
4266                .push(DeliberationRound { round, turns });
4267            self.state.save()?;
4268        }
4269
4270        self.state.status = RunStatus::Voting;
4271        self.state.save()?;
4272        Ok(())
4273    }
4274
4275    // ---------------------------------------------------------------- vote
4276
4277    async fn vote(&mut self) -> Result<()> {
4278        if self.hold_base_sync().await? {
4279            return Ok(());
4280        }
4281        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4282        // agent files with `magi task add` name the run that paid for it. The
4283        // prompt overlay is cloned alongside it because the waves borrow it
4284        // while `self` is mutably borrowed by the node's own bookkeeping.
4285        let run_id = self.state.id.clone();
4286        let prompts = self.state.config.prompts.clone();
4287        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4288            return Ok(());
4289        }
4290        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4291        if viable.len() == 1 {
4292            return Ok(());
4293        }
4294        self.state.status = RunStatus::Voting;
4295
4296        let language = self.state.config.graph.language.clone();
4297        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4298        let sessions = self.state.config.graph.sessions;
4299        let artifacts = agent::artifacts_dir(&self.state.dir());
4300        let root = self.state.worktree_root();
4301        let base_short = short(&self.state.base_commit);
4302        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4303
4304        let mut jobs = Vec::new();
4305        let mut seats_at = Vec::new();
4306        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4307            if self
4308                .state
4309                .judgements
4310                .get(j)
4311                .is_some_and(|r| r.failed.is_some())
4312            {
4313                continue;
4314            }
4315            let seat_key = format!("judge-{}", j + 1);
4316            let spec = self.occupant(&seat_key, spec);
4317            let seat = self.seat(&seat_key, &spec.id);
4318            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4319            let text = if has_context(&spec, &seat, sessions) {
4320                prompt::final_vote(&viable, &language)
4321            } else {
4322                full.clone()
4323            };
4324            jobs.push(SeatJob {
4325                spec,
4326                seat,
4327                prompt: text,
4328                cwd: root.join(format!("judge-{}", j + 1)),
4329                timeout,
4330                allow_write: false,
4331                sessions,
4332                artifacts: artifacts.clone(),
4333                stem: format!("vote-judge-{}", j + 1),
4334                handover: Some(full),
4335            });
4336            seats_at.push(j);
4337        }
4338
4339        self.state.event(
4340            "vote",
4341            format!(
4342                "collecting {} final votes one by one, privately",
4343                jobs.len()
4344            ),
4345        );
4346        let allowed = viable.clone();
4347        let mut quota_losses = Vec::new();
4348        let cache = self.state.config.cache_dir();
4349        let ctx = WaveCtx {
4350            carry_seats: false,
4351            pauses: Some((&self.pause, &self.interrupt)),
4352            run: &run_id,
4353            node: "vote",
4354            prompts: &prompts,
4355            cache: cache.as_deref(),
4356            round: None,
4357        };
4358        let results = ask_json_wave::<FinalVote>(
4359            jobs,
4360            Arc::clone(&self.sem),
4361            self.state.config.graph.retries,
4362            &self.roles.judge_roster,
4363            &ctx,
4364            &mut quota_losses,
4365            &mut self.state,
4366            &move |v: &FinalVote| match v.label() {
4367                Some(c) if allowed.contains(&c) => Ok(()),
4368                other => bail!("vote {other:?} is not one of {allowed:?}"),
4369            },
4370        )
4371        .await;
4372        if self.state.resource_block.is_some() {
4373            return self.stop_for_resource();
4374        }
4375        self.state.quota.extend(quota_losses);
4376
4377        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4378            let agent_id = seat.agent.clone();
4379            self.state.seats.insert(seat.key.clone(), seat);
4380            let initial = self
4381                .state
4382                .judgements
4383                .get(j)
4384                .and_then(|r| r.ranking.first().copied());
4385            let mut record = VoteRecord {
4386                judge: j + 1,
4387                agent: agent_id,
4388                vote: None,
4389                reason: String::new(),
4390                changed: false,
4391            };
4392            match res {
4393                Ok((v, _)) => {
4394                    record.vote = v.label();
4395                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4396                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4397                    self.state.event(
4398                        "vote",
4399                        format!(
4400                            "judge {} voted {}{}",
4401                            j + 1,
4402                            record.vote.unwrap_or('?'),
4403                            if record.changed { " (changed)" } else { "" }
4404                        ),
4405                    );
4406                }
4407                Err(e) => {
4408                    self.state
4409                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4410                }
4411            }
4412            self.state.votes.push(record);
4413            self.state.save()?;
4414        }
4415        Ok(())
4416    }
4417
4418    // --------------------------------------------------------------- tally
4419
4420    fn tally(&mut self) -> Result<()> {
4421        if self.state.tally.is_some() {
4422            return Ok(());
4423        }
4424        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4425        let tops: Vec<char> = self
4426            .state
4427            .judgements
4428            .iter()
4429            .filter_map(|j| j.ranking.first().copied())
4430            .collect();
4431        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4432
4433        // A judge whose private vote failed still counted once, in the initial
4434        // ranking; using it beats discarding a whole seat.
4435        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4436        let mut cast: Vec<char> = Vec::new();
4437        for (i, j) in self.state.judgements.iter().enumerate() {
4438            let vote = self
4439                .state
4440                .votes
4441                .iter()
4442                .find(|v| v.judge == i + 1)
4443                .and_then(|v| v.vote)
4444                .or_else(|| j.ranking.first().copied());
4445            if let Some(v) = vote {
4446                *first_choice.entry(v).or_insert(0) += 1;
4447                cast.push(v);
4448            }
4449        }
4450
4451        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4452        for j in &self.state.judgements {
4453            let n = j.ranking.len();
4454            for (pos, label) in j.ranking.iter().enumerate() {
4455                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4456            }
4457        }
4458
4459        let best = first_choice.values().copied().max().unwrap_or(0);
4460        let mut leaders: Vec<char> = first_choice
4461            .iter()
4462            .filter(|(_, v)| **v == best)
4463            .map(|(k, _)| *k)
4464            .collect();
4465        let mut tie_break = None;
4466        if leaders.len() > 1 {
4467            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4468            let borda_leaders: Vec<char> = leaders
4469                .iter()
4470                .copied()
4471                .filter(|l| borda[l] == top_borda)
4472                .collect();
4473            tie_break = Some(if borda_leaders.len() == 1 {
4474                format!(
4475                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4476                    leaders.len()
4477                )
4478            } else {
4479                format!(
4480                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4481                    leaders.len()
4482                )
4483            });
4484            leaders = borda_leaders;
4485            leaders.sort_unstable();
4486        }
4487        let winner = *leaders
4488            .first()
4489            .or(viable.first())
4490            .context("no candidate to declare a winner from")?;
4491
4492        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4493        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4494        let deliberated = !self.state.deliberation.is_empty();
4495
4496        // Whose verdict is this? A rate-limited seat is absent even if it
4497        // ranked before the limit hit, so presence is measured against the
4498        // recorded losses, not just "did a ranking ever appear".
4499        let quota_seats: std::collections::BTreeSet<&str> =
4500            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4501        let mut present = 0usize;
4502        for (i, j) in self.state.judgements.iter().enumerate() {
4503            if quota_seats.contains(j.seat.as_str()) {
4504                continue;
4505            }
4506            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4507            let voted = self
4508                .state
4509                .votes
4510                .iter()
4511                .any(|v| v.judge == i + 1 && v.vote.is_some());
4512            if ranked || voted {
4513                present += 1;
4514            }
4515        }
4516        // Strict majority of the configured panel. A bare majority is real
4517        // signal we can act on, while a minority verdict must never stand in
4518        // for a healthy one. A one-candidate run needs no panel at all, and
4519        // `judges` stays `0` rather than the roster size a panel that never
4520        // sat would otherwise be credited with.
4521        let needs_quorum = viable.len() > 1;
4522        let judges_total = if needs_quorum {
4523            self.roles.judges.len()
4524        } else {
4525            0
4526        };
4527        let quorum = if needs_quorum {
4528            judges_total / 2 + 1
4529        } else {
4530            0
4531        };
4532        let met_quorum = !needs_quorum || present >= quorum;
4533        let uncontested = (!needs_quorum).then(|| {
4534            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4535        });
4536
4537        self.state.event(
4538            "tally",
4539            match &uncontested {
4540                Some(reason) => format!("winner {winner} — {reason}"),
4541                None => format!(
4542                    "winner {winner} — votes {} | initial {} | {} changed | \
4543                     {present}/{judges_total} judges{}",
4544                    first_choice
4545                        .iter()
4546                        .map(|(k, v)| format!("{k}:{v}"))
4547                        .collect::<Vec<_>>()
4548                        .join(" "),
4549                    if unanimous_initial {
4550                        "unanimous"
4551                    } else {
4552                        "split"
4553                    },
4554                    changed_votes,
4555                    if met_quorum {
4556                        String::new()
4557                    } else {
4558                        format!(" — below quorum ({quorum} required)")
4559                    },
4560                ),
4561            },
4562        );
4563        if !met_quorum {
4564            self.state.event(
4565                "stall",
4566                format!(
4567                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4568                     the run stops here, resumable"
4569                ),
4570            );
4571        }
4572        self.state.tally = Some(Tally {
4573            first_choice,
4574            borda,
4575            winner,
4576            rankings: tops.len(),
4577            unanimous_initial,
4578            deliberated,
4579            changed_votes,
4580            unanimous_final,
4581            tie_break,
4582            judges: judges_total,
4583            present,
4584            quorum,
4585            met_quorum,
4586            uncontested,
4587        });
4588        self.state.status = if met_quorum {
4589            RunStatus::Reviewing
4590        } else {
4591            RunStatus::Stalled
4592        };
4593        self.state.save()?;
4594        Ok(())
4595    }
4596
4597    // ------------------------------------------------------------- recover
4598
4599    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4600    /// resumed toward completion once the transient cause clears.
4601    ///
4602    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4603    /// it toward the quorum, which is exactly the set of seats whose absence
4604    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4605    /// not depend on which node happened to hit the limit), or an ordinary
4606    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4607    /// seat is never disturbed.
4608    ///
4609    /// A seat that now answers with a usable ranking is "recovered": its
4610    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4611    /// `tally` counts it present again), and its vote re-collected. A seat that
4612    /// still fails keeps its loss and stays absent.
4613    ///
4614    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4615    /// to review/gate/merge), `false` when it is still below quorum (the run
4616    /// stays `Stalled`, still resumable for a later retry).
4617    #[allow(clippy::too_many_lines)]
4618    async fn recover_stall(&mut self) -> Result<bool> {
4619        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4620        // agent files with `magi task add` name the run that paid for it. The
4621        // prompt overlay is cloned alongside it because the waves borrow it
4622        // while `self` is mutably borrowed by the node's own bookkeeping.
4623        let run_id = self.state.id.clone();
4624        let prompts = self.state.config.prompts.clone();
4625        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4626        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4627        // is what keeps a non-quota collapse — or a quota loss recorded at the
4628        // deliberate node — from being a permanent dead-end on `--resume`.
4629        let quota_seats: BTreeSet<&str> =
4630            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4631        let absent: Vec<String> = self
4632            .state
4633            .judgements
4634            .iter()
4635            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4636            .map(|j| j.seat.clone())
4637            .collect();
4638        if absent.is_empty() {
4639            return Ok(false);
4640        }
4641        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4642        if viable.len() <= 1 {
4643            return Ok(false);
4644        }
4645        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4646        let language = self.state.config.graph.language.clone();
4647        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4648        let sessions = self.state.config.graph.sessions;
4649        let artifacts = agent::artifacts_dir(&self.state.dir());
4650        let root = self.state.worktree_root();
4651        let base_short = short(&self.state.base_commit);
4652        let candidates: Vec<Candidate> = viable.clone();
4653
4654        // Map each absent seat key to its 0-based position in `roles.judges`.
4655        let mut positions: Vec<usize> = absent
4656            .iter()
4657            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4658            .collect();
4659        if positions.is_empty() {
4660            return Ok(false);
4661        }
4662        positions.sort_unstable();
4663        positions.dedup();
4664
4665        // Re-rank the lost seats, one blind prompt each.
4666        let mut judge_jobs = Vec::new();
4667        for &j in &positions {
4668            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4669            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4670            let seat_key = format!("judge-{}", j + 1);
4671            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4672            let seat = self.seat(&seat_key, &spec.id);
4673            judge_jobs.push(SeatJob {
4674                spec,
4675                seat,
4676                prompt: prompt::judge(
4677                    &self.state.instruction,
4678                    &views,
4679                    self.roles.judges.len(),
4680                    &base_short,
4681                    &language,
4682                ),
4683                cwd: root.join(seat_key),
4684                timeout,
4685                allow_write: false,
4686                sessions,
4687                artifacts: artifacts.clone(),
4688                stem: format!("judge-{}-recover", j + 1),
4689                handover: None,
4690            });
4691        }
4692
4693        let labels_for_check = labels.clone();
4694        let mut judge_losses = Vec::new();
4695        let retries = self.state.config.graph.retries;
4696        let cache = self.state.config.cache_dir();
4697        let ctx = WaveCtx {
4698            carry_seats: false,
4699            pauses: Some((&self.pause, &self.interrupt)),
4700            run: &run_id,
4701            node: "judge",
4702            prompts: &prompts,
4703            cache: cache.as_deref(),
4704            round: None,
4705        };
4706        let results = ask_json_wave::<Ranking>(
4707            judge_jobs,
4708            Arc::clone(&self.sem),
4709            retries,
4710            &self.roles.judge_roster,
4711            &ctx,
4712            &mut judge_losses,
4713            &mut self.state,
4714            &move |r: &Ranking| r.validate(&labels_for_check),
4715        )
4716        .await;
4717        if self.state.resource_block.is_some() {
4718            self.stop_for_resource()?;
4719            return Ok(false);
4720        }
4721
4722        // Refresh the judgement of every seat that ranked again.
4723        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4724        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4725            let agent_id = seat.agent.clone();
4726            self.state.seats.insert(seat.key.clone(), seat);
4727            let record = &mut self.state.judgements[j];
4728            match res {
4729                Ok((ranking, out)) => {
4730                    record.agent = agent_id;
4731                    record.ranking = ranking.normalized();
4732                    record.reasons = ranking.reasons;
4733                    record.strengths = ranking.strengths;
4734                    record.confidence = ranking.confidence;
4735                    record.failed = None;
4736                    record.duration_ms = out.duration_ms;
4737                    recovered.insert(j);
4738                    self.state.event(
4739                        "recover",
4740                        format!("judge {} ranked again after the limit", j + 1),
4741                    );
4742                }
4743                Err(e) => {
4744                    self.state
4745                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4746                }
4747            }
4748        }
4749
4750        // Re-ask the votes of the seats that recovered a ranking.
4751        let mut vote_jobs = Vec::new();
4752        let mut vote_pos: Vec<usize> = Vec::new();
4753        for &j in &recovered {
4754            let seat_key = format!("judge-{}", j + 1);
4755            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4756            let seat = self.seat(&seat_key, &spec.id);
4757            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4758            let text = if has_context(&spec, &seat, sessions) {
4759                prompt::final_vote(&labels, &language)
4760            } else {
4761                full.clone()
4762            };
4763            vote_jobs.push(SeatJob {
4764                spec,
4765                seat,
4766                prompt: text,
4767                cwd: root.join(seat_key),
4768                timeout,
4769                allow_write: false,
4770                sessions,
4771                artifacts: artifacts.clone(),
4772                stem: format!("vote-judge-{}-recover", j + 1),
4773                handover: Some(full),
4774            });
4775            vote_pos.push(j);
4776        }
4777        let allowed = labels.clone();
4778        let mut vote_losses = Vec::new();
4779        let vote_retries = self.state.config.graph.retries;
4780        let vote_cache = self.state.config.cache_dir();
4781        let ctx = WaveCtx {
4782            carry_seats: false,
4783            pauses: Some((&self.pause, &self.interrupt)),
4784            run: &run_id,
4785            node: "vote",
4786            prompts: &prompts,
4787            cache: vote_cache.as_deref(),
4788            round: None,
4789        };
4790        let votes = ask_json_wave::<FinalVote>(
4791            vote_jobs,
4792            Arc::clone(&self.sem),
4793            vote_retries,
4794            &self.roles.judge_roster,
4795            &ctx,
4796            &mut vote_losses,
4797            &mut self.state,
4798            &move |v: &FinalVote| match v.label() {
4799                Some(c) if allowed.contains(&c) => Ok(()),
4800                other => bail!("vote {other:?} is not one of {allowed:?}"),
4801            },
4802        )
4803        .await;
4804        if self.state.resource_block.is_some() {
4805            self.stop_for_resource()?;
4806            return Ok(false);
4807        }
4808        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4809            let agent_id = seat.agent.clone();
4810            self.state.seats.insert(seat.key.clone(), seat);
4811            match res {
4812                Ok((v, _)) => {
4813                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4814                        rec.vote = v.label();
4815                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4816                    } else {
4817                        self.state.votes.push(VoteRecord {
4818                            judge: j + 1,
4819                            agent: agent_id,
4820                            vote: v.label(),
4821                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4822                            changed: false,
4823                        });
4824                    }
4825                    self.state.event(
4826                        "recover",
4827                        format!("judge {} voted again after the limit", j + 1),
4828                    );
4829                }
4830                Err(e) => {
4831                    self.state
4832                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4833                }
4834            }
4835        }
4836
4837        // A seat that ranked again is present even if its re-vote failed —
4838        // `tally` falls back to the initial ranking's first choice — so clear
4839        // its quota loss. Seats that still fail keep theirs and stay absent.
4840        let recovered_keys: BTreeSet<String> = recovered
4841            .iter()
4842            .map(|&j| format!("judge-{}", j + 1))
4843            .collect();
4844        self.state
4845            .quota
4846            .retain(|q| !recovered_keys.contains(&q.seat));
4847        // A seat that hit the limit again is a fresh loss, not the old one:
4848        // replace the stale entry so the history stays one-per-seat and the
4849        // daemon can tell this attempt's loss from a previous session's.
4850        for loss in judge_losses.into_iter().chain(vote_losses) {
4851            if recovered_keys.contains(&loss.seat) {
4852                continue;
4853            }
4854            self.state.quota.retain(|q| q.seat != loss.seat);
4855            self.state.quota.push(loss);
4856        }
4857
4858        // Recompute the verdict from the refreshed panel.
4859        self.state.tally = None;
4860        self.tally()?;
4861        Ok(self
4862            .state
4863            .tally
4864            .as_ref()
4865            .map(|t| t.met_quorum)
4866            .unwrap_or(false))
4867    }
4868
4869    // ---------------------------------------------------------------- graft
4870
4871    /// Port the strengths the judges attributed to the losing candidates onto
4872    /// the winner, once, between the tally and the review loop.
4873    ///
4874    /// Best-effort by construction: it returns nothing and fails nothing. A
4875    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4876    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4877    /// the run carries on as if the phase were off. See [`graft`] for the
4878    /// decisions and `AGENTS.md` for the invariants.
4879    ///
4880    /// The one exception is a rollback that could not be confirmed: the
4881    /// rejected commit may still be the winner's tip, so the run is blocked
4882    /// (resumable; the record stays `Running`, which makes a resume retry the
4883    /// restore) instead of reviewing, gating and merging it.
4884    async fn graft(&mut self) -> Result<()> {
4885        let Err(e) = self.graft_inner().await else {
4886            return Ok(());
4887        };
4888        if e.is::<crate::repo_lock::ResourceBlocked>() {
4889            return Err(e);
4890        }
4891        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4892            self.state.status = RunStatus::Blocked;
4893            self.state.event("graft", format!("blocked: {e:#}"));
4894            self.state.save()?;
4895            return Err(e);
4896        }
4897        // An error after the `Running` record was saved may have left the
4898        // fixer's work in the tree: the record must not stay unsettled
4899        // (a later resume would reset past the review fixes), so confirm the
4900        // restore now, or block.
4901        if let Some(rec) = self.state.graft.clone()
4902            && !rec.outcome.settled()
4903        {
4904            if let Some(winner) = self.state.winner().cloned() {
4905                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4906                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4907                        return Err(re);
4908                    }
4909                    self.state.status = RunStatus::Blocked;
4910                    self.state.event("graft", format!("blocked: {re:#}"));
4911                    self.state.save()?;
4912                    return Err(re);
4913                }
4914            }
4915            if let Some(g) = self.state.graft.as_mut() {
4916                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4917            }
4918        }
4919        self.state.event("graft", format!("skipped: {e:#}"));
4920        if let Err(e) = self.state.save() {
4921            tracing::warn!("save after graft: {e:#}");
4922        }
4923        Ok(())
4924    }
4925
4926    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4927    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4928    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4929        let _coordination = self.coordinate(wt).await?;
4930        crate::recovery::restore_repair(wt, pre)
4931            .await
4932            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4933    }
4934
4935    /// A skip is a run event, recorded once however often the node is re-entered.
4936    fn graft_skipped(&mut self, why: &str) {
4937        let message = format!("skipped: {why}");
4938        if !self
4939            .state
4940            .events
4941            .iter()
4942            .any(|e| e.node == "graft" && e.message == message)
4943        {
4944            self.state.event("graft", message);
4945        }
4946    }
4947
4948    async fn graft_inner(&mut self) -> Result<()> {
4949        // A round that was started and never finished (crash, kill): the
4950        // fixer may have left anything in the tree, so put the winner back.
4951        if let Some(rec) = self.state.graft.clone()
4952            && !rec.outcome.settled()
4953        {
4954            // Reviews exist only if the graft ended without its record being
4955            // settled (a swallowed error): the tree has moved on since, and
4956            // a reset to `pre` would discard the review fixes.
4957            if self.state.reviews.is_empty()
4958                && let Some(winner) = self.state.winner().cloned()
4959            {
4960                self.graft_restore(&winner.worktree, &rec.pre).await?;
4961            }
4962            if rec.outcome == GraftOutcome::WaitingForRestore {
4963                self.state.graft = None;
4964                self.state.save()?;
4965            } else {
4966                if let Some(g) = self.state.graft.as_mut() {
4967                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4968                }
4969                self.state.event(
4970                    "graft",
4971                    "rolled back: the previous round was interrupted; the winner is untouched",
4972                );
4973                self.state.save()?;
4974                return Ok(());
4975            }
4976        }
4977        let plan = match graft::plan(&self.state) {
4978            Ok(plan) => plan,
4979            Err(skip) => {
4980                if skip != graft::Skip::AlreadyDone {
4981                    self.graft_skipped(skip.reason());
4982                    self.state.save()?;
4983                }
4984                return Ok(());
4985            }
4986        };
4987        let Some(winner) = self.state.winner().cloned() else {
4988            return Ok(());
4989        };
4990        let wt = winner.worktree.clone();
4991        if !git::is_clean(&wt).await? {
4992            self.graft_skipped("the winner's worktree has uncommitted changes");
4993            self.state.save()?;
4994            return Ok(());
4995        }
4996        let pre = git::rev_parse(&wt, "HEAD").await?;
4997        // Saved before the fixer is called: a stop from here on is repaired
4998        // by the branch above on the next entry.
4999        self.state.graft = Some(GraftRecord {
5000            pre: pre.clone(),
5001            post: None,
5002            lines: None,
5003            limit: None,
5004            agent: None,
5005            outcome: GraftOutcome::Running,
5006            requested: graft::requested(&plan),
5007            adoption: None,
5008            adoption_error: None,
5009        });
5010        self.state.event(
5011            "graft",
5012            format!(
5013                "porting strengths of candidate(s) {} onto {}",
5014                plan.strengths
5015                    .iter()
5016                    .map(|(l, _, _)| l.to_string())
5017                    .collect::<Vec<_>>()
5018                    .join(", "),
5019                plan.winner
5020            ),
5021        );
5022        self.state.save()?;
5023
5024        let fix_prompt = prompt::graft(
5025            &self.state.instruction,
5026            plan.winner,
5027            &plan.strengths,
5028            &self.state.config.graph.language,
5029        );
5030        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5031        let sessions = self.state.config.graph.sessions;
5032        let artifacts = agent::artifacts_dir(&self.state.dir());
5033        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5034        let (_, seat, out) = self
5035            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5036                SeatJob {
5037                    prompt: fix_prompt.clone(),
5038                    spec,
5039                    seat,
5040                    cwd: wt.clone(),
5041                    timeout,
5042                    allow_write: true,
5043                    sessions,
5044                    artifacts: artifacts.clone(),
5045                    stem: "graft".to_owned(),
5046                    handover: None,
5047                }
5048            })
5049            .await;
5050        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5051            // Preserve the need to restore earlier fallback edits without
5052            // confusing an unstarted seat with an interrupted invocation.
5053            self.state.graft.as_mut().expect("graft record").outcome =
5054                GraftOutcome::WaitingForRestore;
5055            return self.stop_for_resource();
5056        }
5057        let mut why: Option<String> = None;
5058        match &out {
5059            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5060                why = Some("the fixer produced nothing usable".to_owned());
5061            }
5062            AgentOutcome::Ok(_) => {}
5063            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5064            AgentOutcome::Quota(o) => {
5065                self.state.quota.push(QuotaLoss::from_report(
5066                    seat.key.clone(),
5067                    "graft",
5068                    o.quota.as_ref(),
5069                ));
5070                why = Some("rate limited (quota)".to_owned());
5071            }
5072            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5073                why = Some(format!(
5074                    "the fixer failed: {}",
5075                    e.lines()
5076                        .next()
5077                        .unwrap_or("")
5078                        .chars()
5079                        .take(160)
5080                        .collect::<String>()
5081                ));
5082            }
5083        }
5084        let agent_id = seat.agent.clone();
5085        self.state.seats.insert(seat.key.clone(), seat);
5086
5087        let mut post = None;
5088        let mut lines = None;
5089        let _coordination = self.coordinate(&wt).await?;
5090        if why.is_none() {
5091            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5092            if let Ok(r) = &rescued {
5093                self.state.note_withheld("graft", &r.withheld);
5094            }
5095            let head = git::rev_parse(&wt, "HEAD").await?;
5096            // What is measured must be what gets reviewed and gated: a failed
5097            // rescue, or anything still uncommitted (withheld lockfiles
5098            // aside, which are never part of the tree), rejects the graft.
5099            let dirty = match &rescued {
5100                Ok(r) => git::status_porcelain(&wt)
5101                    .await
5102                    .map(|st| {
5103                        st.lines().any(|l| {
5104                            !r.withheld
5105                                .iter()
5106                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5107                        })
5108                    })
5109                    .unwrap_or(true),
5110                Err(_) => true,
5111            };
5112            if dirty {
5113                why = Some("the fixer left changes that could not be committed".to_owned());
5114            } else if head == pre {
5115                why = Some("the fixer changed nothing".to_owned());
5116            } else {
5117                lines = git::changed_lines(&wt, &pre, &head).await;
5118                post = Some(head);
5119            }
5120        }
5121        let (adoption, adoption_error) = match &out {
5122            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5123            _ => (None, None),
5124        };
5125        let requested = graft::requested(&plan);
5126        let outcome = match why {
5127            None => {
5128                self.state.event(
5129                    "graft",
5130                    format!(
5131                        "applied: {}",
5132                        lines.map_or_else(
5133                            || "changed lines unavailable".to_owned(),
5134                            |n| format!("{n} changed lines")
5135                        )
5136                    ),
5137                );
5138                GraftOutcome::Applied
5139            }
5140            Some(why) => {
5141                // Not settled until the restore is confirmed: the record
5142                // stays `Running`, so a resume retries it.
5143                crate::recovery::restore_repair(&wt, &pre)
5144                    .await
5145                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5146                self.state.event(
5147                    "graft",
5148                    format!("rolled back: {why}; the winner is untouched"),
5149                );
5150                GraftOutcome::RolledBack(why)
5151            }
5152        };
5153        self.state.graft = Some(GraftRecord {
5154            pre,
5155            post,
5156            lines,
5157            limit: None,
5158            agent: Some(agent_id),
5159            outcome,
5160            requested,
5161            adoption,
5162            adoption_error,
5163        });
5164        self.state.save()?;
5165        Ok(())
5166    }
5167
5168    // ----------------------------------------------------------------- fold
5169
5170    async fn fold_losers(&mut self) -> Result<()> {
5171        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5172            return Ok(());
5173        };
5174        if !self.state.candidate_preservation.is_empty() {
5175            bail!("candidate preservation incomplete; refusing cleanup");
5176        }
5177        let repo = self.state.repo.clone();
5178        let _coordination = self.coordinate(&repo).await?;
5179        let mut folded = Vec::new();
5180        for i in 0..self.state.candidates.len() {
5181            let c = &self.state.candidates[i];
5182            if c.label == winner || c.folded {
5183                continue;
5184            }
5185            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5186            remove_preserved_worktree(&repo, &wt).await?;
5187            if git::branch_exists(&repo, &branch).await?
5188                && !git::branch_delete(&repo, &branch).await?
5189            {
5190                bail!("candidate branch removal refused");
5191            }
5192            self.state.candidates[i].folded = true;
5193            self.state.save()?;
5194            folded.push(label.to_string());
5195        }
5196        // The judges are finished; their checkouts are pure cost from here.
5197        let root = self.state.worktree_root();
5198        for j in 1..=self.roles.judges.len() {
5199            let wt = root.join(format!("judge-{j}"));
5200            if wt.exists() {
5201                remove_preserved_worktree(&repo, &wt).await?;
5202            }
5203        }
5204        // The design-deliberation stage is finished by the time a tally
5205        // exists — same reasoning as the judges above.
5206        if self.state.config.graph.advise {
5207            for k in 1..=self.state.config.graph.advisors {
5208                let wt = root.join(format!("advisor-{k}"));
5209                if wt.exists() {
5210                    remove_preserved_worktree(&repo, &wt).await?;
5211                }
5212            }
5213        }
5214        if !folded.is_empty() {
5215            self.state
5216                .event("fold", format!("folded candidates {}", folded.join(", ")));
5217            self.state.save()?;
5218        }
5219        Ok(())
5220    }
5221
5222    // ------------------------------------------------------------ base sync
5223
5224    async fn hold_base_sync(&mut self) -> Result<bool> {
5225        self.hold_base_sync_as(None).await
5226    }
5227
5228    async fn hold_base_sync_kind(
5229        &mut self,
5230        kind: crate::base_sync_recovery::StopKind,
5231    ) -> Result<()> {
5232        self.hold_base_sync_as(Some(kind)).await?;
5233        Ok(())
5234    }
5235
5236    async fn hold_base_sync_as(
5237        &mut self,
5238        kind: Option<crate::base_sync_recovery::StopKind>,
5239    ) -> Result<bool> {
5240        if !self.state.base_sync_blocked() {
5241            self.state.base_sync_recovery.stop = None;
5242            return Ok(false);
5243        }
5244        let head = match self.state.winner() {
5245            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5246                .await
5247                .unwrap_or_default(),
5248            None => String::new(),
5249        };
5250        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5251            && !authorization.consumed
5252            && authorization.stop.head != head
5253        {
5254            authorization.refuse(format!(
5255                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5256                authorization.stop.head
5257            ));
5258        }
5259        self.state.record_base_sync_stop(&head);
5260        if let Some(kind) = kind {
5261            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5262            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5263                stop.kind = kind;
5264            }
5265        }
5266        self.state.save()?;
5267        Ok(true)
5268    }
5269
5270    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5271    /// anything verifies it.
5272    ///
5273    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5274    /// read whatever is checked out in the winner's worktree. Left alone that
5275    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5276    /// it when the run *branched* - and a run takes long enough that the base
5277    /// has usually moved by the time it gets here. A gate that ran there
5278    /// answers "green on the commit this run started from", not "green on
5279    /// what is about to land", and the difference showed up three times in
5280    /// one day as a green run whose merge would have reverted a file another
5281    /// pull request had already landed.
5282    ///
5283    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5284    /// `land::Step::Rebase` calls, rather than a second implementation of the
5285    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5286    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5287    /// doc warns about ("two answers to a question nobody notices until a
5288    /// diff is wrong").
5289    ///
5290    /// Foreign-checkout reviews instead merge the base into the owned shadow
5291    /// worktree, preserving the ancestry required for fast-forward delivery.
5292    /// A conflicting merge is aborted and held for operator resolution.
5293    ///
5294    /// A conflict is not the end of the road: the standing rebase is handed
5295    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5296    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5297    /// land). Once it finishes, review and the gate run as usual on the
5298    /// rebased tree, which is where a breakage the new base caused is caught
5299    /// by the ordinary gate-fix round. magi resolves nothing itself.
5300    ///
5301    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5302    /// the current durable recovery interval, bounds how often the base is
5303    /// synchronized during active recovery (lifetime attempts remain evidence).
5304    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5305    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5306    /// what was tried (rounds spent, paths still conflicted) and the branch
5307    /// and worktree stay exactly as they were - untouched, for a person to
5308    /// look at - which is also what makes re-entering this function
5309    /// afterwards a no-op instead of a second attempt at the same wall. A
5310    /// push failure ends the same way.
5311    async fn sync_to_base(&mut self) -> Result<()> {
5312        if self.state.status == RunStatus::AlreadyInBase {
5313            return Ok(());
5314        }
5315        self.hold_base_sync().await?;
5316        let conflicted = self
5317            .state
5318            .base_sync
5319            .as_ref()
5320            .is_some_and(|s| s.conflict.is_some());
5321        let Some(winner) = self.state.winner().cloned() else {
5322            return Ok(());
5323        };
5324
5325        let repo = self.state.repo.clone();
5326        let remote = self.state.config.merge.remote.clone();
5327        let base_branch = self.state.base_branch.clone();
5328        let tracking = format!("{remote}/{base_branch}");
5329
5330        let fetched_base =
5331            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5332        // No network, or the remote never had this branch: the run already
5333        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5334        // otherwise), and one that got this far is not blocked by a fetch
5335        // that fails now. It just has no newer tip to compare against.
5336        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5337            return Ok(());
5338        };
5339
5340        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5341        self.state.ensure_base_episode();
5342        let mut pending = self
5343            .state
5344            .base_sync_recovery
5345            .operations
5346            .last()
5347            .filter(|o| !o.finished)
5348            .cloned();
5349        let scratch = self.state.dir().join("base-sync");
5350        if git::rebase_in_progress(&scratch).await {
5351            // Older resource stops predate operation reservations. Import the
5352            // standing Git operation, never retarget it to today's main.
5353            if pending.is_none()
5354                && self.state.base_sync_recovery.operations.is_empty()
5355                && !conflicted
5356                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5357                    .await
5358                    .ok()
5359                    .as_deref()
5360                    == Some(head.as_str())
5361                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5362                && git::is_clean(&winner.worktree).await?
5363            {
5364                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5365                if self.state.base_sync.is_none() {
5366                    self.state.base_sync = Some(BaseSync {
5367                        tip: saved_tip.clone(),
5368                        behind: 0,
5369                        attempts: 0,
5370                        conflict: None,
5371                        already_in: None,
5372                    });
5373                }
5374                self.state.reserve_base_operation(&head, &saved_tip);
5375                self.state.event(
5376                    "base-recovery",
5377                    "imported standing legacy rebase as a spent synchronization reservation",
5378                );
5379                self.state.save()?;
5380                pending = self.state.base_sync_recovery.operations.last().cloned();
5381            }
5382            let expected = pending.as_ref().map(|op| op.tip.as_str());
5383            if expected.is_none()
5384                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5385            {
5386                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5387                if self.state.base_sync.is_none() {
5388                    self.state.base_sync = Some(BaseSync {
5389                        tip: tip.clone(),
5390                        behind: 0,
5391                        attempts: 0,
5392                        conflict: None,
5393                        already_in: None,
5394                    });
5395                }
5396                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5397                self.state.event("land", why);
5398                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5399                    .await?;
5400                return Ok(());
5401            }
5402        }
5403        if let Some(op) = &pending {
5404            // A replay may add the pinned base's commits, never foreign work.
5405            let mut replay_proven =
5406                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5407            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5408                let wait = repository_wait(&self.state, None);
5409                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5410                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5411                    Ok(true) => {
5412                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5413                        self.state.save()?;
5414                        git::worktree_remove(&repo, &scratch).await.ok();
5415                        replay_proven = true;
5416                    }
5417                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5418                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5419                }
5420            }
5421            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5422                if self.state.foreign_checkout.is_some() {
5423                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5424                        .await
5425                        .ok()
5426                        .as_deref()
5427                        == Some(op.head.as_str())
5428                        && git::rev_parse(&repo, &format!("{head}^2"))
5429                            .await
5430                            .ok()
5431                            .as_deref()
5432                            == Some(op.tip.as_str());
5433                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5434                    .await
5435                    .is_ok_and(|m| m.is_empty())
5436                    && let Ok(added) =
5437                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5438                {
5439                    replay_proven = true;
5440                    for commit in added {
5441                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5442                            replay_proven = false;
5443                            break;
5444                        }
5445                    }
5446                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5447                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5448                }
5449            }
5450            // Never retarget an operation across a crash, even if main moved.
5451            // sync_to_head resets and cleans with -x, so even ignored untracked
5452            // files must stop recovery independently of the old tracked tree.
5453            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5454                || (head != op.head
5455                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5456                        .await?
5457                        .is_empty())
5458                || (!git::is_clean(&winner.worktree).await?
5459                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5460                        .await
5461                        .is_ok_and(|o| o.ok()))
5462                || !replay_proven
5463            {
5464                let why = format!(
5465                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5466                    op.id
5467                );
5468                let sync = self
5469                    .state
5470                    .base_sync
5471                    .as_mut()
5472                    .context("reserved synchronization has no lifetime accounting")?;
5473                sync.conflict = Some(why.clone());
5474                self.state.event("land", why);
5475                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5476                    .await?;
5477                return Ok(());
5478            }
5479            if head != op.head {
5480                git::sync_to_head(&winner.worktree).await?;
5481            }
5482            tip = op.tip.clone();
5483        }
5484        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5485        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5486        if pending.is_none()
5487            && !conflicted
5488            && fetched_base
5489            && self
5490                .state
5491                .base_sync_recovery
5492                .episodes
5493                .last()
5494                .is_some_and(|e| e.boundary.is_some())
5495            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5496            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5497            && owner.runs.last() == Some(&self.state.id)
5498            && owner.status != crate::queue::TaskStatus::Done
5499            && !owner.fresh_start
5500            && owner.review_branch.is_none()
5501            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5502            && let Some(approval) = owner.resumption.as_ref()
5503            && approval.instruction == owner.instruction
5504            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5505            && git::is_clean(&winner.worktree).await?
5506            && self.state.renew_base_episode(&task, approval, &head, &tip)
5507        {
5508            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5509            self.state.save()?;
5510        }
5511
5512        // Before any rebase, and before a recorded conflict is honoured: a
5513        // branch whose change reached the base under other commit ids has
5514        // nothing to rebase and nothing to conflict with, and a run that
5515        // already stopped on that phantom conflict recovers here on resume.
5516        // `behind == 0` with head == tip is a branch the base has since taken
5517        // in whole, whether or not a conflict was ever recorded: the ancestry
5518        // proof must still run (`classify` ignores a head still on the start
5519        // commit).
5520        if (behind > 0 || conflicted || head == tip)
5521            && self
5522                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5523                .await?
5524        {
5525            return Ok(());
5526        }
5527        if conflicted && self.state.base_recovery_pending() {
5528            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5529            let owner =
5530                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5531            let owner_valid = owner.as_ref().is_ok_and(|t| {
5532                t.runs.last() == Some(&self.state.id)
5533                    && t.status != crate::queue::TaskStatus::Done
5534                    && !t.fresh_start
5535                    && t.review_branch.is_none()
5536            });
5537            let invalidation = if !owner_valid {
5538                Some("latest task ownership changed")
5539            } else if !owner.as_ref().is_ok_and(|t| {
5540                authorization
5541                    .generation
5542                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5543            }) {
5544                Some("owner approval generation changed")
5545            } else if let Some(scope) = &authorization.scope {
5546                if scope.tip != tip {
5547                    Some("fetched base changed")
5548                } else if !self
5549                    .state
5550                    .base_sync_recovery
5551                    .episodes
5552                    .last()
5553                    .is_some_and(|e| e.id == scope.episode)
5554                {
5555                    Some("active synchronization interval changed")
5556                } else if !owner
5557                    .as_ref()
5558                    .is_ok_and(|t| t.instruction == scope.instruction)
5559                {
5560                    Some("task instruction changed")
5561                } else {
5562                    None
5563                }
5564            } else {
5565                None
5566            };
5567            if let Some(reason) = invalidation {
5568                let authorization = self
5569                    .state
5570                    .base_sync_recovery
5571                    .authorizations
5572                    .last_mut()
5573                    .unwrap();
5574                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5575                self.state.save()?;
5576                return Ok(());
5577            }
5578            if !fetched_base
5579                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5580                || !git::is_clean(&winner.worktree).await?
5581                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5582            {
5583                return Ok(());
5584            }
5585        }
5586        let recovery = conflicted && self.state.consume_base_recovery(&head);
5587        if conflicted {
5588            // Consumption (including a refused changed HEAD) is durable before Git.
5589            self.state.save()?;
5590            if !recovery {
5591                return Ok(());
5592            }
5593        }
5594
5595        if behind == 0 {
5596            // A fixer-finished rebase moves the branch ref before the
5597            // winner's worktree is told (`sync_to_head` below). A run that
5598            // died in between resumes here with `behind == 0` and a tree still
5599            // holding the pre-rebase files, which review and the gate would
5600            // then read. That state is exactly: HEAD moved off the tip the
5601            // rebase started from, yet the tree is still identical to that
5602            // tip. A tree with edits of its own differs from it, so nothing
5603            // is thrown away.
5604            if let Some(from) = self
5605                .state
5606                .rebase_fixes
5607                .iter()
5608                .rev()
5609                .find_map(|r| r.from.clone())
5610                && from != head
5611                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5612                    .await
5613                    .is_ok_and(|o| o.ok())
5614            {
5615                git::sync_to_head(&winner.worktree).await?;
5616            }
5617            // An earlier attempt may have rebased the branch locally and died
5618            // before pushing it (only the fresh-rebase arm below pushes).
5619            // Publish it now, so the plain push at PR time is not refused as
5620            // a non-fast-forward. A run already holding a recorded conflict
5621            // never reaches here; that case is out of scope.
5622            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5623            if let Some(why) = &conflict {
5624                self.state.status = RunStatus::Blocked;
5625                self.state.event("land", why.clone());
5626            }
5627            self.state.base_sync = Some(BaseSync {
5628                tip,
5629                behind: 0,
5630                attempts,
5631                conflict,
5632                already_in: None,
5633            });
5634            if pending.is_some() {
5635                self.state.finish_base_operation(Some(&head));
5636            }
5637            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5638                .await?;
5639            self.state.save()?;
5640            return Ok(());
5641        }
5642
5643        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5644            let why = format!(
5645                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5646                 synchronization(s); synchronizing again would only race it",
5647                winner.branch
5648            );
5649            self.state.status = RunStatus::Blocked;
5650            self.state.base_sync = Some(BaseSync {
5651                tip,
5652                behind,
5653                attempts,
5654                conflict: Some(why.clone()),
5655                already_in: None,
5656            });
5657            self.state.event("land", why);
5658            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5659                .await?;
5660            self.state.save()?;
5661            return Ok(());
5662        }
5663
5664        if self.state.foreign_checkout.is_some() {
5665            // Delivery to the operator's checkout is fast-forward only. A
5666            // rebase here would discard its saved head from our ancestry.
5667            let wait = repository_wait(&self.state, None);
5668            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5669            if self.state.base_sync.is_none() {
5670                self.state.base_sync = Some(BaseSync {
5671                    tip: tip.clone(),
5672                    behind,
5673                    attempts: 0,
5674                    conflict: None,
5675                    already_in: None,
5676                });
5677            }
5678            self.state.reserve_base_operation(&head, &tip);
5679            self.state.save()?;
5680            let merged = git::git_raw(
5681                &winner.worktree,
5682                &[
5683                    "-c",
5684                    "core.hooksPath=/dev/null",
5685                    "merge",
5686                    "--no-ff",
5687                    "--no-autostash",
5688                    "--no-edit",
5689                    "-m",
5690                    &format!("Merge {tracking} before verifying"),
5691                    &tip,
5692                ],
5693            )
5694            .await?;
5695            let conflict = if merged.ok() {
5696                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5697                self.state.event(
5698                    "land",
5699                    format!("merged {tracking} into {} before verifying", winner.branch),
5700                );
5701                None
5702            } else {
5703                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5704                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5705                }
5706                let why = format!(
5707                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5708                    winner.branch,
5709                    merged.stderr.chars().take(600).collect::<String>()
5710                );
5711                self.state.status = RunStatus::Blocked;
5712                self.state.event("land", &why);
5713                Some(why)
5714            };
5715            let synced = conflict.is_none();
5716            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5717            self.state
5718                .finish_base_operation(synced.then_some(fresh.as_str()));
5719            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5720            self.state.base_sync = Some(BaseSync {
5721                tip,
5722                behind: if synced { 0 } else { behind },
5723                attempts,
5724                conflict,
5725                already_in: None,
5726            });
5727            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5728                .await?;
5729            self.state.save()?;
5730            return Ok(());
5731        }
5732
5733        self.state.event(
5734            "land",
5735            format!(
5736                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5737                winner.branch
5738            ),
5739        );
5740        self.state.save()?;
5741
5742        // The remote's copy of the branch, read now and only if the fetch
5743        // really succeeded (a stale tracking ref must never pin a lease). It is
5744        // pushed over after a rebase only when it is a commit this branch
5745        // already contains, by ancestry or by patch (an earlier rebase of ours
5746        // that never reached the remote): anything else is somebody else's work.
5747        let branch_tracking = format!("{remote}/{}", winner.branch);
5748        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5749        let remote_tip = if self.state.task_pr_branch().is_none()
5750            && matches!(&fetched_branch, Ok(o) if o.ok())
5751        {
5752            git::rev_parse(&repo, &branch_tracking).await.ok()
5753        } else {
5754            None
5755        };
5756        // A remote tip this branch does not contain is somebody else's work:
5757        // rebasing would leave a local tip that can never be pushed. Stop
5758        // before touching anything and say so.
5759        if let Some(theirs) = &remote_tip
5760            && !git::is_ancestor(&repo, theirs, &head).await
5761            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5762                .await
5763                .is_ok_and(|missing| missing.is_empty())
5764        {
5765            let why = format!(
5766                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5767                 them",
5768                short(theirs),
5769                winner.branch
5770            );
5771            self.state.status = RunStatus::Blocked;
5772            self.state.base_sync = Some(BaseSync {
5773                tip,
5774                behind,
5775                attempts,
5776                conflict: Some(why.clone()),
5777                already_in: None,
5778            });
5779            self.state.event("land", why);
5780            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5781                .await?;
5782            self.state.save()?;
5783            return Ok(());
5784        }
5785
5786        if self.state.base_sync.is_none() {
5787            self.state.base_sync = Some(BaseSync {
5788                tip: tip.clone(),
5789                behind,
5790                attempts: 0,
5791                conflict: None,
5792                already_in: None,
5793            });
5794        }
5795        self.state.reserve_base_operation(&head, &tip);
5796        self.state.save()?;
5797        let scratch = self.state.dir().join("base-sync");
5798        let rebased =
5799            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5800                .await
5801            {
5802                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5803                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5804                Err(e) => Err(e),
5805            };
5806        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5807        let stop_kind = match &rebased {
5808            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5809            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5810            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5811        };
5812        match rebased {
5813            Ok(None) => {
5814                // The branch ref moved, but a worktree that already had it
5815                // checked out (the winner's) was not told; sync its index and
5816                // files before anything reads them.
5817                git::sync_to_head(&winner.worktree).await?;
5818                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5819                self.state.finish_base_operation(Some(&fresh));
5820                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5821                let mut conflict = None;
5822                if let Some(pinned) = &remote_tip {
5823                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5824                    match pushed {
5825                        Ok(o) if o.ok() => self.state.event(
5826                            "land",
5827                            format!("pushed rebased {} to {remote}", winner.branch),
5828                        ),
5829                        Ok(o) => {
5830                            conflict = Some(format!(
5831                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5832                                winner.branch,
5833                                short(pinned),
5834                                o.stderr.chars().take(600).collect::<String>()
5835                            ));
5836                        }
5837                        Err(e) => {
5838                            conflict = Some(format!(
5839                                "rebased {} locally but could not push it: {e:#}",
5840                                winner.branch
5841                            ));
5842                        }
5843                    }
5844                }
5845                if let Some(why) = &conflict {
5846                    self.state.status = RunStatus::Blocked;
5847                    self.state.event("land", why.clone());
5848                }
5849                self.state.base_sync = Some(BaseSync {
5850                    tip: tip.clone(),
5851                    behind: 0,
5852                    attempts,
5853                    conflict,
5854                    already_in: None,
5855                });
5856                self.state
5857                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5858            }
5859            Ok(Some(conflict)) => {
5860                self.state.finish_base_operation(None);
5861                let why = format!(
5862                    "{} conflicts with {tracking} and did not rebase: {}",
5863                    winner.branch,
5864                    conflict.chars().take(600).collect::<String>()
5865                );
5866                self.state.status = RunStatus::Blocked;
5867                self.state.base_sync = Some(BaseSync {
5868                    tip,
5869                    behind,
5870                    attempts,
5871                    conflict: Some(why.clone()),
5872                    already_in: None,
5873                });
5874                self.state.event("land", why);
5875            }
5876            // A refused Git write or a busy repository: nothing is wrong with
5877            // the branch. The scratch rebase stays standing and the run
5878            // resumes into it (`base_sync` must not claim a conflict).
5879            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5880                self.state.save()?;
5881                return Ok(());
5882            }
5883            Err(e) => {
5884                self.state.finish_base_operation(None);
5885                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5886                self.state.status = RunStatus::Blocked;
5887                self.state.base_sync = Some(BaseSync {
5888                    tip,
5889                    behind,
5890                    attempts,
5891                    conflict: Some(why.clone()),
5892                    already_in: None,
5893                });
5894                self.state.event("land", why);
5895            }
5896        }
5897        if !self.state.base_sync_blocked() {
5898            self.state.base_sync_recovery.stop = None;
5899        }
5900        self.hold_base_sync_kind(stop_kind).await?;
5901        self.state.save()?;
5902        Ok(())
5903    }
5904
5905    /// Push a branch an earlier attempt rebased locally but never published,
5906    /// pinned to the remote tip read right after a successful fetch. Returns
5907    /// the reason when the run must stop; `None` when there was nothing to do
5908    /// (no remote copy, the same tip, or a remote copy this branch already
5909    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5910    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5911        // Task competitions publish only after the PR text gate, to their
5912        // public head; never resurrect a candidate branch on the remote.
5913        if self.state.task_pr_branch().is_some() {
5914            return None;
5915        }
5916        let repo = self.state.repo.clone();
5917        let remote = self.state.config.merge.remote.clone();
5918        let fetched = git::fetch(&repo, &remote, branch).await;
5919        if !matches!(&fetched, Ok(o) if o.ok()) {
5920            return None;
5921        }
5922        let branch_tracking = format!("{remote}/{branch}");
5923        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5924        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5925            return None;
5926        }
5927        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5928            .await
5929            .is_ok_and(|missing| missing.is_empty())
5930        {
5931            return Some(format!(
5932                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5933                 them",
5934                short(&theirs)
5935            ));
5936        }
5937        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5938            Ok(o) if o.ok() => {
5939                self.state
5940                    .event("land", format!("pushed rebased {branch} to {remote}"));
5941                None
5942            }
5943            Ok(o) => Some(format!(
5944                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5945                 someone may have pushed): {}",
5946                short(&theirs),
5947                o.stderr.chars().take(600).collect::<String>()
5948            )),
5949            Err(e) => Some(format!(
5950                "{branch} is rebased locally but could not be pushed: {e:#}"
5951            )),
5952        }
5953    }
5954
5955    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5956    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5957    /// whether it did.
5958    ///
5959    /// Checked only when the base is ahead of the branch. A failing check is
5960    /// "not proven" - the ordinary rebase path then decides - never a reason to
5961    /// stop the run.
5962    ///
5963    /// The remote copy of the branch is held to the same standard as the local
5964    /// one: if it carries a tip this worktree does not, that tip must itself be
5965    /// proven in the base, or nothing is settled (a pull request would
5966    /// otherwise be closed over commits nobody checked). The pull request is
5967    /// closed *before* the terminal status is saved; if that fails for a
5968    /// reason other than a refusal (no network, a `gh` error) the run is left
5969    /// `Blocked` with the reason as its conflict, which a resume retries -
5970    /// the same recovery a phantom conflict gets.
5971    async fn settle_already_in(
5972        &mut self,
5973        branch: &str,
5974        tip: &str,
5975        head: &str,
5976        attempts: usize,
5977        behind: usize,
5978    ) -> Result<bool> {
5979        let repo = self.state.repo.clone();
5980        let remote = self.state.config.merge.remote.clone();
5981        let start = self.state.base_commit.clone();
5982        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5983            Ok(Some(e)) => e,
5984            Ok(None) => return Ok(false),
5985            Err(e) => {
5986                tracing::warn!("already-in-base check for {branch}: {e:#}");
5987                return Ok(false);
5988            }
5989        };
5990        let mut verified = vec![head.to_owned()];
5991        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5992        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5993        if matches!(&fetched, Ok(o) if o.ok())
5994            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5995            && theirs != head
5996        {
5997            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5998                Ok(Some(_)) => verified.push(theirs),
5999                _ => return Ok(false),
6000            }
6001        }
6002        let base_branch = self.state.base_branch.clone();
6003        let message = format!(
6004            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6005             land",
6006            evidence.names(),
6007            evidence.proof.as_str()
6008        );
6009        let closed =
6010            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6011                .await;
6012        match closed {
6013            Ok(Ok(url)) => self
6014                .state
6015                .event("land", format!("closed {url}: superseded on {base_branch}")),
6016            Ok(Err(why)) => self
6017                .state
6018                .event("land", format!("did not close a pull request: {why}")),
6019            Err(e) => {
6020                let why = format!(
6021                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6022                     not be closed ({e:#}); resume to retry"
6023                );
6024                self.state.status = RunStatus::Blocked;
6025                self.state.base_sync = Some(BaseSync {
6026                    tip: tip.to_owned(),
6027                    behind,
6028                    attempts,
6029                    conflict: Some(why.clone()),
6030                    already_in: None,
6031                });
6032                self.state.event("land", why);
6033                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6034                    .await?;
6035                self.state.save()?;
6036                return Ok(true);
6037            }
6038        }
6039        self.state.status = RunStatus::AlreadyInBase;
6040        self.state.base_sync_recovery.stop = None;
6041        self.state.base_sync = Some(BaseSync {
6042            tip: tip.to_owned(),
6043            behind,
6044            attempts,
6045            conflict: None,
6046            already_in: Some(evidence),
6047        });
6048        self.state.event("land", message);
6049        self.state.save()?;
6050        self.settle_questions();
6051        Ok(true)
6052    }
6053
6054    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6055    /// last landed the winner on, once it has run, else the commit the run
6056    /// branched from.
6057    ///
6058    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6059    /// and `vote` all happen before there is a winner to rebase, so they
6060    /// compare every candidate against the branch point on purpose, and a
6061    /// base that moves after they are already done cannot change an answer
6062    /// they already gave.
6063    fn landing_base(&self) -> String {
6064        self.state
6065            .base_sync
6066            .as_ref()
6067            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6068    }
6069
6070    // ------------------------------------------------------- operator fix
6071
6072    /// Route specific, already-recorded review findings to a fixer for a
6073    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6074    /// entry point.
6075    ///
6076    /// Distinct from `review_loop`'s own fix step in three ways: it never
6077    /// runs a reviewer wave, it never spends review-round budget, and what
6078    /// happened is recorded as an [`OperatorFixRequest`] appended to
6079    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6080    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6081    /// and vote must never be rewritten to look like a manufactured blocking
6082    /// verdict.
6083    ///
6084    /// Only meaningful once review has actually concluded: `Ready` (handed
6085    /// off with findings still open, or simply concluded clean while minor
6086    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6087    /// gate failed). Everything else is refused: a run still in progress
6088    /// should simply be resumed, and a `Merged` run's branch has already
6089    /// landed — reopening *this* run's own record cannot change that, so the
6090    /// answer there is a fresh `magi review <branch>`.
6091    ///
6092    /// A real commit here re-verifies through a fresh, ordinary review-only
6093    /// run on the same branch ([`Self::review`]) rather than reopening this
6094    /// run's own `review_loop`: once any round in this run's history went
6095    /// clean, `review_conclusion` treats that as permanent by design (the
6096    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6097    /// way to force one more genuine reviewer wave out of *this* run without
6098    /// either rewriting history or weakening that guarantee for every other
6099    /// caller. A review-only run costs nothing extra — no implementation, no
6100    /// judging, no vote — and exercises the exact same review → verify →
6101    /// gate → (human) merge path, unmodified.
6102    pub async fn fix_selected(
6103        &mut self,
6104        ids: &[String],
6105        reason: &str,
6106        allow_stale: bool,
6107    ) -> Result<()> {
6108        let reason = reason.trim();
6109        if reason.is_empty() {
6110            bail!("a fix request needs a reason — that is the operator's own record of why");
6111        }
6112        if ids.is_empty() {
6113            bail!("no finding id given");
6114        }
6115        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6116            bail!(
6117                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6118                 has already concluded — can be given a targeted fix. A run still \
6119                 in progress should simply be resumed; a `merged` run's branch has \
6120                 already landed, so its answer is a fresh `magi review <branch>`, \
6121                 not reopening this run's own record",
6122                self.state.id,
6123                self.state.status.as_str()
6124            );
6125        }
6126        let Some(winner) = self.state.winner().cloned() else {
6127            bail!("run {} has no winning candidate to fix", self.state.id);
6128        };
6129        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6130            bail!(
6131                "branch `{}` no longer exists; this run cannot be extended",
6132                winner.branch
6133            );
6134        }
6135        let home = crate::run::home();
6136        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6137            bail!(
6138                "run {} is currently being worked on by another magi process",
6139                self.state.id
6140            );
6141        }
6142        // Held for the rest of this call, including the follow-up review
6143        // below: two `magi fix` invocations against the same run must not
6144        // both reach the worktree manipulation further down, which would
6145        // otherwise race to remove and recreate the same directory — see
6146        // [`FixClaim`]'s own doc.
6147        let _claim = FixClaim::acquire(&self.state.dir())?;
6148
6149        // Resolve every id before spending anything — an unknown id refuses
6150        // the whole request rather than silently dropping it — and dedup
6151        // while keeping the operator's own order.
6152        let mut seen = BTreeSet::new();
6153        let mut findings = Vec::new();
6154        let mut missing = Vec::new();
6155        for id in ids {
6156            if !seen.insert(id.clone()) {
6157                continue;
6158            }
6159            match self.state.finding(id) {
6160                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6161                    id: f.id.clone(),
6162                    severity: f.severity,
6163                    reviewer_vote: rec.vote,
6164                    round: round.round,
6165                    round_head: round.head.clone(),
6166                    reviewer: rec.reviewer,
6167                    agent: rec.agent.clone(),
6168                    file: f.file.clone(),
6169                    line: f.line,
6170                    title: f.title.clone(),
6171                    detail: f.detail.clone(),
6172                    outcome: OperatorFixOutcome::Pending,
6173                }),
6174                None => missing.push(id.clone()),
6175            }
6176        }
6177        if !missing.is_empty() {
6178            bail!(
6179                "unknown finding id(s): {}; nothing was changed",
6180                missing.join(", ")
6181            );
6182        }
6183
6184        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6185        let stale_details: Vec<(String, String)> = findings
6186            .iter()
6187            .filter(|f| f.round_head != head_at_request)
6188            .map(|f| (f.id.clone(), f.round_head.clone()))
6189            .collect();
6190        let stale = !stale_details.is_empty();
6191        if stale && !allow_stale {
6192            bail!(
6193                "the branch has moved since some finding(s) were raised — {} — now \
6194                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6195                stale_details
6196                    .iter()
6197                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6198                    .collect::<Vec<_>>()
6199                    .join(", "),
6200                short(&head_at_request)
6201            );
6202        }
6203
6204        let request = OperatorFixRequest {
6205            requested_at: Timestamp::now(),
6206            reason: reason.to_owned(),
6207            findings,
6208            head_at_request: head_at_request.clone(),
6209            allow_stale,
6210            stale,
6211            fix: None,
6212            result_head: None,
6213            follow_up_review_run: None,
6214        };
6215        self.state.event(
6216            "fix",
6217            format!(
6218                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6219                request.findings.len(),
6220                request
6221                    .findings
6222                    .iter()
6223                    .map(|f| f.id.as_str())
6224                    .collect::<Vec<_>>()
6225                    .join(", "),
6226            ),
6227        );
6228        // Recorded now, before any worktree work or the fixer call itself —
6229        // and re-saved at each checkpoint below: a crash at any point after
6230        // this (mid fixer call, mid follow-up review) must not lose the fact
6231        // that this was requested, for which findings, and why. Everything
6232        // past this point reads and writes through `request_index` rather
6233        // than a local variable, since `request` itself is moved here.
6234        self.state.operator_fixes.push(request);
6235        self.state.save()?;
6236        let request_index = self.state.operator_fixes.len() - 1;
6237
6238        // A fresh, dedicated worktree for this one call, never the winner's
6239        // own worktree in place: that one may already be gone (folded away),
6240        // and reusing it in place would leave the branch checked out there
6241        // when the follow-up review below tries to check it out again. Freed
6242        // immediately after, either way — but only once confirmed clean:
6243        // `worktree_remove` is a `git worktree remove --force`, which would
6244        // otherwise discard uncommitted work left there by the operator or
6245        // another process before this had a chance to even look at it.
6246        let repo = self.state.repo.clone();
6247        let coordination = self.coordinate(&repo).await?;
6248        if winner.worktree.exists() {
6249            // Lockfiles a rescue commit withheld stay untracked on purpose and
6250            // are already recorded; they are not the operator's work to protect.
6251            let dirty = git::git(
6252                &winner.worktree,
6253                &["status", "--porcelain", "--untracked-files=all"],
6254            )
6255            .await?;
6256            let only_withheld = dirty.lines().all(|l| {
6257                l.strip_prefix("?? ")
6258                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6259            });
6260            if !only_withheld {
6261                bail!(
6262                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6263                     discard them first",
6264                    winner.worktree.display()
6265                );
6266            }
6267            git::worktree_remove(&self.state.repo, &winner.worktree)
6268                .await
6269                .ok();
6270        }
6271        let fix_worktree = self.state.worktree_root().join("operator-fix");
6272        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6273        git::git(
6274            &self.state.repo,
6275            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6276        )
6277        .await
6278        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6279        if !git::is_clean(&fix_worktree).await? {
6280            git::worktree_remove(&self.state.repo, &fix_worktree)
6281                .await
6282                .ok();
6283            bail!(
6284                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6285                winner.branch
6286            );
6287        }
6288        let repo_for_setup = self.state.repo.clone();
6289        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6290            Ok(()) => {
6291                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6292            }
6293            Err(e) => Err(e),
6294        };
6295        if let Err(e) = setup {
6296            git::worktree_remove(&self.state.repo, &fix_worktree)
6297                .await
6298                .ok();
6299            return Err(e);
6300        }
6301
6302        // This explicit operator request retries availability independently
6303        // of execute(); the previous call has already saved and cleaned up.
6304        self.state.resource_block = None;
6305
6306        let run_id = self.state.id.clone();
6307        let prompts = self.state.config.prompts.clone();
6308        let language = self.state.config.graph.language.clone();
6309        let sessions = self.state.config.graph.sessions;
6310        let artifacts = agent::artifacts_dir(&self.state.dir());
6311        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6312            .findings
6313            .iter()
6314            .map(|f| Finding {
6315                id: f.id.clone(),
6316                severity: f.severity,
6317                file: f.file.clone(),
6318                line: f.line,
6319                title: f.title.clone(),
6320                detail: f.detail.clone(),
6321            })
6322            .collect();
6323        let fix_prompt = prompt::operator_fix_in(
6324            &self.state.instruction,
6325            &finding_list,
6326            reason,
6327            &stale_details,
6328            &head_at_request,
6329            &language,
6330            &self.state.config.graph.github_language,
6331        );
6332        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6333        drop(coordination);
6334        let (job, seat, out) = self
6335            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6336                prompt: fix_prompt.clone(),
6337                spec,
6338                seat,
6339                cwd: fix_worktree.clone(),
6340                timeout,
6341                allow_write: true,
6342                sessions,
6343                artifacts: artifacts.clone(),
6344                stem: "operator-fix".to_owned(),
6345                handover: None,
6346            })
6347            .await;
6348        let agent_id = seat.agent.clone();
6349
6350        let mut fix = FixRecord {
6351            agent: agent_id,
6352            addressed: Vec::new(),
6353            rejected: Vec::new(),
6354            notes: String::new(),
6355            committed: false,
6356            failed: None,
6357            duration_ms: 0,
6358            continuation: None,
6359        };
6360        let mut final_seat = seat.clone();
6361        match out {
6362            AgentOutcome::Ok(o) => {
6363                fix.duration_ms = o.duration_ms;
6364                let parsed = verdict::extract_json::<FixReport>(&o.text);
6365                let incomplete_reason = match &parsed {
6366                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6367                        "the reply parsed, but it reported a command whose own CLI \
6368                         never confirmed an exit status"
6369                            .to_owned(),
6370                    ),
6371                    Ok(_) => None,
6372                    Err(e) => Some(e.to_string()),
6373                };
6374                match incomplete_reason {
6375                    None => {
6376                        let report = parsed.expect("checked Ok above");
6377                        fix.addressed = report.addressed;
6378                        fix.rejected = report.rejected;
6379                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6380                    }
6381                    Some(reason) => {
6382                        let (resumed_seat, resolved, failure, cont) = self
6383                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6384                            .await;
6385                        fix.duration_ms += cont.cumulative_wait_ms;
6386                        fix.continuation = Some(cont);
6387                        final_seat = resumed_seat;
6388                        match resolved {
6389                            Some(report) => {
6390                                fix.addressed = report.addressed;
6391                                fix.rejected = report.rejected;
6392                                fix.notes =
6393                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6394                            }
6395                            None => fix.failed = failure,
6396                        }
6397                    }
6398                }
6399            }
6400            AgentOutcome::Dropped(o) => {
6401                fix.duration_ms = o.duration_ms;
6402                let why = o
6403                    .dropped
6404                    .as_ref()
6405                    .map(|d| d.why.as_str())
6406                    .unwrap_or("the CLI ended the stream without delivering its answer");
6407                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6408            }
6409            AgentOutcome::Quota(o) => {
6410                self.state.quota.push(QuotaLoss::from_report(
6411                    final_seat.key.clone(),
6412                    "fix",
6413                    o.quota.as_ref(),
6414                ));
6415                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6416            }
6417            AgentOutcome::Failed(e) => fix.failed = Some(e),
6418            AgentOutcome::ResourceBlocked(e) => {
6419                fix.failed = Some(format!("the fixer was not started: {e}"));
6420            }
6421        }
6422        // Even a deferred call must rescue edits, release the worktree and
6423        // finalize its record below so an explicit `magi fix` can retry.
6424        if fix.continuation.is_none() {
6425            fix.continuation = Some(ContinuationRecord::not_needed());
6426        }
6427        self.state.seats.insert(final_seat.key.clone(), final_seat);
6428
6429        let rescue_message = format!(
6430            "magi: operator-selected fix ({}) (uncommitted work)",
6431            self.state.operator_fixes[request_index]
6432                .findings
6433                .iter()
6434                .map(|f| f.id.as_str())
6435                .collect::<Vec<_>>()
6436                .join(", ")
6437        );
6438        let repo = self.state.repo.clone();
6439        let coordination = self.coordinate(&repo).await?;
6440        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6441            self.state.note_withheld("fix", &r.withheld);
6442        }
6443        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6444        fix.committed = after != head_at_request;
6445        git::worktree_remove(&self.state.repo, &fix_worktree)
6446            .await
6447            .ok();
6448        drop(coordination);
6449
6450        self.state.event(
6451            "fix",
6452            match &fix.failed {
6453                Some(reason) => format!(
6454                    "operator fix: adoption report was lost ({reason}); {}",
6455                    if fix.committed {
6456                        "committed"
6457                    } else {
6458                        "NO new commit"
6459                    }
6460                ),
6461                None => format!(
6462                    "operator fix: {} addressed, {} rejected, {}",
6463                    fix.addressed.len(),
6464                    fix.rejected.len(),
6465                    if fix.committed {
6466                        "committed"
6467                    } else {
6468                        "NO new commit"
6469                    }
6470                ),
6471            },
6472        );
6473
6474        // Every selected finding gets an outcome — never left `Pending` once
6475        // the fixer's own turn is over. A report that never came back at all
6476        // marks every one of them `Unreported`, not silently "not addressed":
6477        // quota, a dropped stream, or an exhausted continuation are gaps in
6478        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6479        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6480        for f in &mut self.state.operator_fixes[request_index].findings {
6481            f.outcome = if fix.failed.is_some() {
6482                OperatorFixOutcome::Unreported
6483            } else if fix.addressed.contains(&f.id) {
6484                OperatorFixOutcome::Addressed
6485            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6486                OperatorFixOutcome::Rejected { why: r.why.clone() }
6487            } else {
6488                OperatorFixOutcome::Unreported
6489            };
6490        }
6491
6492        let committed = fix.committed;
6493        if committed {
6494            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6495        }
6496        self.state.operator_fixes[request_index].fix = Some(fix);
6497        // Saved again now that the fixer's own outcome is final, on top of
6498        // the save right after the request was first pushed above.
6499        self.state.save()?;
6500        if self.state.resource_block.is_some() {
6501            return self.stop_for_resource();
6502        }
6503
6504        if committed {
6505            self.state.event(
6506                "fix",
6507                format!(
6508                    "operator fix committed {}; opening a follow-up review-only run",
6509                    short(&after)
6510                ),
6511            );
6512            // The operator asked for the fix, and the follow-up serves whatever
6513            // task the run it follows served.
6514            let origin =
6515                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6516            match Self::review(
6517                &self.state.repo,
6518                &winner.branch,
6519                self.state.config.clone(),
6520                origin,
6521            )
6522            .await
6523            {
6524                Ok(mut follow_up) => {
6525                    follow_up.state.event(
6526                        "start",
6527                        format!(
6528                            "requested by an operator fix on run {} for finding(s) {}",
6529                            self.state.id,
6530                            self.state.operator_fixes[request_index]
6531                                .findings
6532                                .iter()
6533                                .map(|f| f.id.as_str())
6534                                .collect::<Vec<_>>()
6535                                .join(", "),
6536                        ),
6537                    );
6538                    follow_up.state.save()?;
6539                    let follow_up_id = follow_up.state.id.clone();
6540                    // The follow-up is a run like any other: it belongs to the
6541                    // task of the run it follows, or to one filed for it.
6542                    let adopted = match crate::direct::adopt(
6543                        &follow_up.state,
6544                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6545                    ) {
6546                        Ok(a) => a,
6547                        Err(e) => {
6548                            // An ownerless run must not spend agent calls; it
6549                            // stays saved, and `magi run --resume` adopts it.
6550                            self.state.event(
6551                                "fix",
6552                                format!(
6553                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6554                                ),
6555                            );
6556                            self.state.save()?;
6557                            return Ok(());
6558                        }
6559                    };
6560                    let executed = follow_up.execute().await;
6561                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6562                    if let Some(a) = adopted {
6563                        a.finish(&follow_up.state, executed);
6564                    }
6565                    if let Some(e) = failure {
6566                        self.state.event(
6567                            "fix",
6568                            format!(
6569                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6570                            ),
6571                        );
6572                    }
6573                    self.state.operator_fixes[request_index].follow_up_review_run =
6574                        Some(follow_up_id);
6575                }
6576                Err(e) => {
6577                    self.state.event(
6578                        "fix",
6579                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6580                    );
6581                }
6582            }
6583            self.state.save()?;
6584        }
6585
6586        Ok(())
6587    }
6588
6589    // --------------------------------------------------------------- review
6590
6591    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6592    /// names, in order, each at most once, moving on only when the call
6593    /// advances (an error, a quota hit or nothing usable - the same decision
6594    /// point `agent::chain_advances` is for the other chained roles).
6595    ///
6596    /// Each agent gets its own seat from `Runner::seat`: the same agent
6597    /// continues its conversation (the winner's own implementer seat when it
6598    /// is the winner's author, now that the competition is over), another
6599    /// takes a fresh one so the full prompt is sent again. A handover is
6600    /// recorded under `node`, which is what makes the fallback stick for the
6601    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6602    /// is returned, so an exhausted chain reads exactly like a single failed
6603    /// fixer: one quota loss, the same wording, the same refund. An earlier
6604    /// attempt's edits are left in the tree and judged, with the final
6605    /// attempt's, by what git says afterwards.
6606    ///
6607    /// Returns the job that produced the outcome, for `continue_fix_report`.
6608    async fn ask_fixer(
6609        &mut self,
6610        winner: &Candidate,
6611        node: &'static str,
6612        round: Option<usize>,
6613        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6614    ) -> (SeatJob, SeatState, AgentOutcome) {
6615        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6616        self.ask_fixer_among(attempts, node, round, None, build)
6617            .await
6618    }
6619
6620    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6621    /// worktree and a commit: before the next agent is asked after one that
6622    /// did not satisfy the call, the worktree is put back on that commit, so
6623    /// a failed agent's partial edits are not carried into the next one.
6624    async fn ask_fixer_among(
6625        &mut self,
6626        attempts: Vec<(AgentSpec, String)>,
6627        node: &'static str,
6628        round: Option<usize>,
6629        restore: Option<(&Path, &str)>,
6630        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6631    ) -> (SeatJob, SeatState, AgentOutcome) {
6632        let run_id = self.state.id.clone();
6633        let prompts = self.state.config.prompts.clone();
6634        let cache = self.state.config.cache_dir();
6635        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6636        let mut last = None;
6637        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6638            let seat = self.seat(&key, &spec.id);
6639            let mut job = build(spec.clone(), seat);
6640            if i > 0 {
6641                job.stem = format!("{}-{}", job.stem, spec.id);
6642            }
6643            let ctx = WaveCtx {
6644                carry_seats: false,
6645                pauses: Some((&self.pause, &self.interrupt)),
6646                run: &run_id,
6647                node,
6648                prompts: &prompts,
6649                cache: cache.as_deref(),
6650                round,
6651            };
6652            let (seat, out) =
6653                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6654            // A resource block never advances the chain: the chosen agent
6655            // was not asked, so it must be the one asked again on resume.
6656            let advances = match &out {
6657                AgentOutcome::Ok(o) => agent::output_advances(o),
6658                AgentOutcome::ResourceBlocked(_) => false,
6659                _ => true,
6660            };
6661            if let Some(next) = ids.get(i + 1)
6662                && advances
6663            {
6664                self.state.seats.insert(seat.key.clone(), seat.clone());
6665                let class =
6666                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6667                let reason = match &out {
6668                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6669                    other => fail_reason(other),
6670                };
6671                // A failed agent's edits must not reach the next one, and a
6672                // restore that did not work ends the chain: the call fails
6673                // and the caller's own (confirmed) restore decides.
6674                if let Some((wt, rev)) = restore
6675                    && let Err(e) = self.graft_restore(wt, rev).await
6676                {
6677                    let why =
6678                        format!("could not restore the worktree before the next fixer: {e:#}");
6679                    tracing::warn!("{why}");
6680                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6681                        AgentOutcome::ResourceBlocked(why)
6682                    } else {
6683                        AgentOutcome::Failed(why)
6684                    };
6685                    last = Some((job, seat, out));
6686                    break;
6687                }
6688                record_handover(
6689                    &mut self.state,
6690                    node,
6691                    &seat.key,
6692                    &spec.id,
6693                    next,
6694                    &class,
6695                    &reason,
6696                );
6697                continue;
6698            }
6699            last = Some((job, seat, out));
6700            break;
6701        }
6702        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6703        if let AgentOutcome::Quota(output) = &out
6704            && node != "graft"
6705        {
6706            // Graft is best effort: its caller records the loss and restores
6707            // the pre-graft tree, then proceeds to review and gate.
6708            // Every fallback was tried. Machine outages cannot be repaired by
6709            // spending a fix round; use each caller's resumable resource path.
6710            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6711            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6712            self.state.quota.push(loss);
6713            self.state.seats.insert(seat.key.clone(), seat.clone());
6714            self.state.status = RunStatus::Blocked;
6715            return (
6716                job,
6717                seat,
6718                AgentOutcome::ResourceBlocked(
6719                    "fixer unavailable; restore agent availability and resume".into(),
6720                ),
6721            );
6722        }
6723        (job, seat, out)
6724    }
6725
6726    /// The one repair a verification-only failure earns (see
6727    /// [`crate::recovery`]), before the first review round of a run that took
6728    /// a branch over.
6729    ///
6730    /// Saved as started *before* the fixer is called, so a resume never
6731    /// repairs twice; a start found on re-entry is rolled back to the commit
6732    /// it began from. A failed or unusable fixer leaves the tree as it was and
6733    /// the review goes on over it, so the first round's own verification
6734    /// reports the failure again. The failure is never carried into a round as
6735    /// a result: verification is re-run on whatever commit this leaves.
6736    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6737        use crate::recovery::{Mode, Repair};
6738        if !self.state.reviews.is_empty() {
6739            return Ok(());
6740        }
6741        let Some(mut rec) = self.state.recovery.clone() else {
6742            return Ok(());
6743        };
6744        if rec.mode != Mode::VerifyRepair {
6745            return Ok(());
6746        }
6747        let wt = winner.worktree.clone();
6748        let mut head = git::rev_parse(&wt, "HEAD").await?;
6749        match rec.repair.clone() {
6750            Repair::Done { .. } => return Ok(()),
6751            Repair::WaitingForRestore { before } => {
6752                self.graft_restore(&wt, &before).await?;
6753                head = before;
6754                rec.repair = Repair::NotStarted;
6755                self.state.recovery = Some(rec.clone());
6756                self.state.save()?;
6757            }
6758            Repair::Started { before } => {
6759                // Interrupted: the unfinished edits are not trusted.
6760                // An unconfirmed rollback blocks the run with `Started` kept,
6761                // so a resume retries it; the tree is never reviewed as is.
6762                if let Err(e) = self.graft_restore(&wt, &before).await {
6763                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6764                        return Err(e);
6765                    }
6766                    return self
6767                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6768                        .await;
6769                }
6770                rec.repair = Repair::Done {
6771                    before: before.clone(),
6772                    after: before,
6773                    committed: false,
6774                    failed: Some("interrupted; rolled back".to_owned()),
6775                };
6776                self.state
6777                    .event("recovery", "the repair was interrupted and rolled back");
6778                self.state.recovery = Some(rec);
6779                self.state.save()?;
6780                return Ok(());
6781            }
6782            Repair::NotStarted => {}
6783        }
6784        if crate::recovery::recheck(&mut rec, &head) {
6785            self.state.event("recovery", rec.summary());
6786            self.state.recovery = Some(rec);
6787            self.state.save()?;
6788            return Ok(());
6789        }
6790        rec.repair = Repair::Started {
6791            before: head.clone(),
6792        };
6793        self.state.recovery = Some(rec.clone());
6794        self.state.save()?;
6795
6796        let language = self.state.config.graph.language.clone();
6797        let sessions = self.state.config.graph.sessions;
6798        let artifacts = agent::artifacts_dir(&self.state.dir());
6799        let fix_prompt = prompt::recovery_fix(
6800            &self.state.instruction,
6801            &rec,
6802            self.state.config.graph.review_rounds,
6803            &language,
6804        );
6805        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6806        self.state.event(
6807            "recovery",
6808            format!(
6809                "repair: the fixer starts from the failed verification on {}",
6810                short(&head)
6811            ),
6812        );
6813        let (_job, seat, out) = self
6814            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6815                prompt: fix_prompt.clone(),
6816                spec,
6817                seat,
6818                cwd: wt.clone(),
6819                timeout,
6820                allow_write: true,
6821                sessions,
6822                artifacts: artifacts.clone(),
6823                stem: "fix-recovery".to_owned(),
6824                handover: None,
6825            })
6826            .await;
6827        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6828            rec.repair = Repair::WaitingForRestore { before: head };
6829            self.state.recovery = Some(rec);
6830            return self.stop_for_resource();
6831        }
6832        let failed = match &out {
6833            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6834            AgentOutcome::Quota(o) => {
6835                self.state.quota.push(QuotaLoss::from_report(
6836                    seat.key.clone(),
6837                    "fix",
6838                    o.quota.as_ref(),
6839                ));
6840                Some("rate limited (quota); the fixer could not run".to_owned())
6841            }
6842            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6843            AgentOutcome::Failed(e) => Some(e.clone()),
6844        };
6845        self.state.seats.insert(seat.key.clone(), seat);
6846        let coordination = self.coordinate(&wt).await?;
6847        let failed = match failed {
6848            // The agent's partial edits are not kept.
6849            Some(why) => {
6850                if let Err(e) = git::restore_to(&wt, &head).await {
6851                    return self
6852                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6853                        .await;
6854                }
6855                Some(why)
6856            }
6857            None => {
6858                if let Ok(r) = git::rescue_commit(
6859                    &wt,
6860                    "magi: recovery repair of the failed verification (uncommitted work)",
6861                )
6862                .await
6863                {
6864                    self.state.note_withheld("fix", &r.withheld);
6865                }
6866                None
6867            }
6868        };
6869        let after = git::rev_parse(&wt, "HEAD").await?;
6870        drop(coordination);
6871        self.state.event(
6872            "recovery",
6873            match &failed {
6874                Some(why) => format!("repair: nothing kept ({why})"),
6875                None if after != head => {
6876                    format!("repair: committed {}", short(&after))
6877                }
6878                None => "repair: the fixer made no commit".to_owned(),
6879            },
6880        );
6881        rec.repair = Repair::Done {
6882            before: head.clone(),
6883            committed: after != head,
6884            after,
6885            failed,
6886        };
6887        self.state.recovery = Some(rec);
6888        self.state.save()?;
6889        Ok(())
6890    }
6891
6892    /// A repair whose rollback could not be confirmed: the run is blocked and
6893    /// resumable, and the unconfirmed state is never reviewed.
6894    async fn stop_recovery(
6895        &mut self,
6896        mut rec: crate::recovery::RecoveryRecord,
6897        head: &str,
6898        why: String,
6899    ) -> Result<()> {
6900        // `Started` stays recorded: a resume retries the restore.
6901        rec.repair = crate::recovery::Repair::Started {
6902            before: head.to_owned(),
6903        };
6904        self.state.recovery = Some(rec);
6905        self.state
6906            .event("recovery", format!("repair could not be undone: {why}"));
6907        self.state.status = RunStatus::Blocked;
6908        self.state.save()?;
6909        bail!("recovery repair could not be rolled back: {why}")
6910    }
6911
6912    async fn review_loop(&mut self) -> Result<()> {
6913        if self.hold_base_sync().await? {
6914            return Ok(());
6915        }
6916        loop {
6917            if self
6918                .state
6919                .base_sync
6920                .as_ref()
6921                .is_some_and(|s| s.conflict.is_some())
6922            {
6923                return Ok(());
6924            }
6925            if self.catchup_repairable() {
6926                let record = self.state.reviews.last().expect("repairable round").clone();
6927                let winner = self
6928                    .state
6929                    .winner()
6930                    .cloned()
6931                    .context("catch-up repair has no winner")?;
6932                if !self.fix_review_round(&winner, record).await? {
6933                    return Ok(());
6934                }
6935            }
6936            self.review_loop_pass().await?;
6937            if !self.catchup_repairable() {
6938                return Ok(());
6939            }
6940        }
6941    }
6942
6943    /// Derive an owed catch-up repair from durable round evidence, including
6944    /// reentry after a crash before the fixer was dispatched.
6945    fn catchup_repairable(&self) -> bool {
6946        self.state.reviews.len() < self.state.config.graph.review_rounds
6947            && self.state.reviews.last().is_some_and(|r| {
6948                !r.clean
6949                    && r.blocking == 0
6950                    && r.fix.is_none()
6951                    && r.e2e_status() == E2eStatus::Failed
6952            })
6953    }
6954
6955    async fn review_loop_pass(&mut self) -> Result<()> {
6956        // A base that would not rebase is a person's decision, not a review
6957        // round: nothing here would change the answer, and reviewers and a
6958        // fixer would be spending real budget on a tree that cannot land
6959        // regardless of what they find.
6960        if self
6961            .state
6962            .base_sync
6963            .as_ref()
6964            .is_some_and(|s| s.conflict.is_some())
6965        {
6966            return Ok(());
6967        }
6968        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6969        // agent files with `magi task add` name the run that paid for it. The
6970        // prompt overlay is cloned alongside it because the waves borrow it
6971        // while `self` is mutably borrowed by the node's own bookkeeping.
6972        let run_id = self.state.id.clone();
6973        let prompts = self.state.config.prompts.clone();
6974        let Some(winner) = self.state.winner().cloned() else {
6975            return Ok(());
6976        };
6977        let granted_review = self
6978            .state
6979            .repair_grants
6980            .last()
6981            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
6982            .cloned();
6983        let fresh_review = granted_review
6984            .as_ref()
6985            .is_some_and(|g| self.state.reviews.len() == g.review_start);
6986        let max_rounds = granted_review
6987            .as_ref()
6988            .map_or(self.state.config.graph.review_rounds, |g| {
6989                g.review_start.saturating_add(1)
6990            });
6991        // A clean round, an exhausted round budget, or a stalled tree (see
6992        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6993        // they are recorded — recomputed here, not read off `status`, so a
6994        // reentry into a run that already stopped restates the identical
6995        // verdict instead of silently handing back whatever an earlier node
6996        // in this same walk clobbered `status` to (a solo-candidate
6997        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6998        // below runs an empty range once the budget is spent, and would
6999        // otherwise fall through without touching `status` at all.
7000        if let Some(status) =
7001            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7002        {
7003            // A reentry after a crash between the last round's save and
7004            // `stop_reviewing` reaches the hand-off here, not there.
7005            if status == RunStatus::Gating {
7006                self.record_contested_handoff();
7007            }
7008            self.state.status = status;
7009            self.state.save()?;
7010            return Ok(());
7011        }
7012        self.state.status = RunStatus::Reviewing;
7013        // A last recorded round whose own verification never resolved
7014        // (`ResourceBlocked` — the shared build cache, not the patch) is
7015        // never a concluded round, whatever the round budget says: starting
7016        // a fresh round on top of it would spend a whole new reviewer wave
7017        // re-reading an unchanged patch instead of just retrying the one
7018        // check that actually needs it, and once the budget is spent the
7019        // loop below has nothing left to do at all (its range is empty).
7020        // Retry that check directly instead, exactly the same retry
7021        // `stop_reviewing` already does for its own catch-up case.
7022        if !fresh_review
7023            && self.state.reviews.last().is_some_and(|r| {
7024                r.e2e_status() == E2eStatus::ResourceBlocked
7025                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7026            })
7027        {
7028            let shell = self.state.config.shell();
7029            return self
7030                .stop_unresolved_review(
7031                    "the last round's own verification never resolved",
7032                    &shell,
7033                    &winner.worktree,
7034                )
7035                .await;
7036        }
7037
7038        if granted_review.is_none() {
7039            self.recovery_repair(&winner).await?;
7040        }
7041        if self.resource_halt()? {
7042            return Ok(());
7043        }
7044
7045        let repo = self.state.repo.clone();
7046        let root = self.state.worktree_root();
7047        let language = self.state.config.graph.language.clone();
7048        let sessions = self.state.config.graph.sessions;
7049        let artifacts = self.merge_repair_artifacts();
7050        let base = self.landing_base();
7051        let base_short = short(&base);
7052        let reviewers = self.roles.reviewers.clone();
7053        let shell = self.state.config.shell();
7054
7055        for round in (self.state.reviews.len() + 1)..=max_rounds {
7056            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7057            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7058            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7059            // The prior round's own record, already persisted — never a
7060            // hand-carried variable of just its failing output: that is
7061            // exactly what let a round's e2e result drift out of sync with
7062            // which commit it was actually about (see `SCHEMA`'s doc for
7063            // schema 8). Judged against `head`, the commit reviewers are
7064            // about to look at now, so the summary always reads as "an
7065            // earlier head" here — this round's own patch has not been
7066            // checked yet.
7067            let prev_verification = self
7068                .state
7069                .reviews
7070                .last()
7071                .and_then(|r| r.verification_summary(&head));
7072            // Only the first round of a run that took a branch over: later
7073            // rounds have this run's own history.
7074            let prior_section = match (&self.state.recovery, round) {
7075                (Some(rec), 1) if self.state.reviews.is_empty() => {
7076                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7077                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7078                            Some(crate::recovery::head_of(
7079                                &git::diff(&winner.worktree, reviewed, "HEAD")
7080                                    .await
7081                                    .unwrap_or_default(),
7082                                crate::recovery::SINCE_DIFF_MAX,
7083                            ))
7084                        }
7085                        _ => None,
7086                    };
7087                    Some(prompt::recovery_review(rec, since.as_deref()))
7088                }
7089                _ => None,
7090            };
7091
7092            // Each reviewer gets its own detached checkout of exactly this
7093            // commit: nobody can perturb the winner's tree, and the fixer can
7094            // keep working without racing a reviewer.
7095            let mut jobs = Vec::new();
7096            let coordination = self.coordinate(&repo).await?;
7097            for (r, spec) in reviewers.iter().cloned().enumerate() {
7098                let wt = root.join(format!("review-{}", r + 1));
7099                // `clean -fdx` in a reset wipes setup's products, so every
7100                // round sets the seat up again.
7101                if wt.exists() {
7102                    git::reset_detached(&wt, &head).await?;
7103                } else {
7104                    git::worktree_add_detached(&repo, &wt, &head).await?;
7105                }
7106                self.setup_seat_worktree(&repo, &wt, false).await?;
7107                let seat_key = format!("review-{}", r + 1);
7108                // The seat starts the round on whoever answered it last, not
7109                // on the agent the spec names, so a failure is not re-paid.
7110                let spec = pick_start_spec(
7111                    &self.roles.reviewer_roster,
7112                    spec,
7113                    self.state.seat_history.get(&seat_key),
7114                );
7115                let seat = self.seat(&seat_key, &spec.id);
7116                jobs.push(SeatJob {
7117                    prompt: prompt::review(&prompt::ReviewCtx {
7118                        instruction: &self.state.instruction,
7119                        branch: &winner.branch,
7120                        base_short: &base_short,
7121                        stat: &stat,
7122                        patch: &patch,
7123                        verification: prev_verification.as_ref(),
7124                        prior: prior_section.as_deref(),
7125                        reviewers: reviewers.len(),
7126                        round,
7127                        rounds: max_rounds,
7128                        // A review-only run has no rankings, so nothing
7129                        // competed for this patch and the reviewer is told so.
7130                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7131                        lens: Lens::for_seat(r),
7132                        language: &language,
7133                        github_language: &self.state.config.graph.github_language,
7134                    }),
7135                    spec,
7136                    seat,
7137                    cwd: wt,
7138                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7139                    allow_write: false,
7140                    sessions,
7141                    artifacts: artifacts.clone(),
7142                    stem: format!("review-{round}-{}", r + 1),
7143                    handover: None,
7144                });
7145            }
7146
7147            drop(coordination);
7148            self.state.event(
7149                "review",
7150                format!(
7151                    "round {round}: {} reviewers on {}",
7152                    jobs.len(),
7153                    short(&head)
7154                ),
7155            );
7156            let mut quota_losses = Vec::new();
7157            let review_retries = self.state.config.graph.retries;
7158            let review_cache = self.state.config.cache_dir();
7159            let ctx = WaveCtx {
7160                carry_seats: true,
7161                pauses: Some((&self.pause, &self.interrupt)),
7162                run: &run_id,
7163                node: "review",
7164                prompts: &prompts,
7165                cache: review_cache.as_deref(),
7166                round: Some(round),
7167            };
7168            let results = ask_json_wave::<Review>(
7169                jobs,
7170                Arc::clone(&self.sem),
7171                review_retries,
7172                &self.roles.reviewer_roster,
7173                &ctx,
7174                &mut quota_losses,
7175                &mut self.state,
7176                &|_: &Review| Ok(()),
7177            )
7178            .await;
7179            if self.state.resource_block.is_some() {
7180                return self.stop_for_resource();
7181            }
7182            // Counted before the move below: how many of *this* round's
7183            // reviewer seats were lost to their own rate limit, as opposed to
7184            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7185            let round_quota_missing = quota_losses.len();
7186            self.state.quota.extend(quota_losses);
7187
7188            let mut records = Vec::new();
7189            let mut all_findings = Vec::new();
7190            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7191                let agent_id = seat.agent.clone();
7192                self.state.seats.insert(seat.key.clone(), seat);
7193                let mut record = ReviewRecord {
7194                    reviewer: r + 1,
7195                    agent: agent_id,
7196                    summary: String::new(),
7197                    findings: Vec::new(),
7198                    vote: None,
7199                    failed: None,
7200                    duration_ms: 0,
7201                    // Set for both outcomes: `failed: Some(_)` with
7202                    // `attempts > 0` is a seat every retry still lost, not a
7203                    // recovered one — only `failed: None` with `attempts > 0`
7204                    // reads as "answered after a nudge" (see this field's own
7205                    // doc).
7206                    attempts,
7207                };
7208                match res {
7209                    Ok((review, out)) => {
7210                        // Sanitized here, at the point every other piece of
7211                        // agent prose in this file is (candidate summaries,
7212                        // deliberation turns, vote reasons): a reviewer's own
7213                        // words are the one thing about it that could name
7214                        // it, and reconsideration below broadcasts this same
7215                        // summary and these same findings to every other
7216                        // seat on the panel.
7217                        record.summary =
7218                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7219                        record.vote = Some(review.vote);
7220                        record.duration_ms = out.duration_ms;
7221                        for (n, mut f) in review.findings.into_iter().enumerate() {
7222                            // ids are magi's, never the agent's: the fixer's
7223                            // adoption report is keyed by them.
7224                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7225                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7226                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7227                            // `file` is agent-supplied prose too, never
7228                            // checked against the real tree — the same
7229                            // exposure `title`/`detail` above have, just in
7230                            // a field easy to forget because it looks like a
7231                            // path rather than free text.
7232                            f.file = f
7233                                .file
7234                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7235                            all_findings.push(f.clone());
7236                            record.findings.push(f);
7237                        }
7238                        self.state.event(
7239                            "review",
7240                            format!(
7241                                "round {round}: reviewer {} voted {} with {} finding(s)",
7242                                r + 1,
7243                                review.vote.label(),
7244                                record.findings.len()
7245                            ),
7246                        );
7247                    }
7248                    Err(e) => {
7249                        record.failed = Some(e.to_string());
7250                        self.state.event(
7251                            "review",
7252                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7253                        );
7254                    }
7255                }
7256                records.push(record);
7257            }
7258
7259            // Tally the round's votes and, if they split, spend the one
7260            // round of reconsideration the split -> deliberate -> revote
7261            // shape `judge`/`vote` use for the panel, sized down to what a
7262            // read-only review round can afford: one round, and a revote
7263            // rather than an argument, because the panel already wrote its
7264            // reasoning down as findings the first time around.
7265            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7266            let vote_split =
7267                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7268            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7269            if vote_split {
7270                self.state.event(
7271                    "review",
7272                    format!(
7273                        "round {round}: votes split ({}) — one round of reconsideration",
7274                        initial_votes
7275                            .iter()
7276                            .map(|v| v.label())
7277                            .collect::<Vec<_>>()
7278                            .join(", ")
7279                    ),
7280                );
7281                // Seats read every seat's findings and votes, still numbered
7282                // and never named — the same anonymity `review` itself keeps.
7283                let panel: Vec<ReviewSeatReport<'_>> = records
7284                    .iter()
7285                    .filter_map(|r| {
7286                        r.vote.map(|vote| ReviewSeatReport {
7287                            reviewer: r.reviewer,
7288                            vote,
7289                            summary: &r.summary,
7290                            findings: &r.findings,
7291                        })
7292                    })
7293                    .collect();
7294
7295                let mut jobs = Vec::new();
7296                let mut seats_at = Vec::new();
7297                for (r, spec) in reviewers.iter().cloned().enumerate() {
7298                    // A seat with no initial vote has nothing to reconsider
7299                    // from and stays absent, the same as it stayed absent
7300                    // from `panel` above.
7301                    if records[r].vote.is_none() {
7302                        continue;
7303                    }
7304                    let wt = root.join(format!("review-{}", r + 1));
7305                    let seat_key = format!("review-{}", r + 1);
7306                    let spec = self.occupant(&seat_key, spec);
7307                    let seat = self.seat(&seat_key, &spec.id);
7308                    // A seat with no live session has already forgotten the
7309                    // initial review's prompt — restate the patch it is
7310                    // voting on, the same as `deliberate`/`vote` do for a
7311                    // judge in the same position.
7312                    // The panel already carries this seat's own review and
7313                    // vote, so restating the patch makes the prompt whole for
7314                    // a seat handed to another agent.
7315                    let build = |with_patch: bool| {
7316                        prompt::review_reconsider(&ReviewReconsiderCtx {
7317                            instruction: &self.state.instruction,
7318                            reviewer: r + 1,
7319                            lens: Lens::for_seat(r),
7320                            panel: &panel,
7321                            patch: with_patch.then_some(ReviewPatch {
7322                                branch: &winner.branch,
7323                                base_short: &base_short,
7324                                stat: &stat,
7325                                patch: &patch,
7326                            }),
7327                            round,
7328                            rounds: max_rounds,
7329                            language: &language,
7330                        })
7331                    };
7332                    let full = build(true);
7333                    let prompt = if has_context(&spec, &seat, sessions) {
7334                        build(false)
7335                    } else {
7336                        full.clone()
7337                    };
7338                    jobs.push(SeatJob {
7339                        prompt,
7340                        spec,
7341                        seat,
7342                        cwd: wt,
7343                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7344                        allow_write: false,
7345                        sessions,
7346                        artifacts: artifacts.clone(),
7347                        stem: format!("review-{round}-reconsider-{}", r + 1),
7348                        handover: Some(full),
7349                    });
7350                    seats_at.push(r);
7351                }
7352
7353                let mut recon_quota_losses = Vec::new();
7354                let recon_cache = self.state.config.cache_dir();
7355                let recon_ctx = WaveCtx {
7356                    carry_seats: true,
7357                    pauses: Some((&self.pause, &self.interrupt)),
7358                    run: &run_id,
7359                    node: "review",
7360                    prompts: &prompts,
7361                    cache: recon_cache.as_deref(),
7362                    round: Some(round),
7363                };
7364                let recon_results = ask_json_wave::<ReviewRevote>(
7365                    jobs,
7366                    Arc::clone(&self.sem),
7367                    review_retries,
7368                    &self.roles.reviewer_roster,
7369                    &recon_ctx,
7370                    &mut recon_quota_losses,
7371                    &mut self.state,
7372                    &|_: &ReviewRevote| Ok(()),
7373                )
7374                .await;
7375                if self.state.resource_block.is_some() {
7376                    return self.stop_for_resource();
7377                }
7378                self.state.quota.extend(recon_quota_losses);
7379
7380                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7381                    let agent_id = seat.agent.clone();
7382                    self.state.seats.insert(seat.key.clone(), seat);
7383                    let mut rec = ReviewRevoteRecord {
7384                        reviewer: r + 1,
7385                        agent: agent_id,
7386                        vote: None,
7387                        reason: String::new(),
7388                        failed: None,
7389                    };
7390                    match res {
7391                        Ok((rv, _)) => {
7392                            rec.vote = Some(rv.vote);
7393                            rec.reason =
7394                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7395                            self.state.event(
7396                                "review",
7397                                format!(
7398                                    "round {round}: reviewer {} revoted {}",
7399                                    r + 1,
7400                                    rv.vote.label()
7401                                ),
7402                            );
7403                        }
7404                        Err(e) => {
7405                            rec.failed = Some(e.to_string());
7406                            self.state.event(
7407                                "review",
7408                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7409                            );
7410                        }
7411                    }
7412                    reconsideration.push(rec);
7413                }
7414            } else if initial_votes.len() > 1 {
7415                self.state.event(
7416                    "review",
7417                    format!(
7418                        "round {round}: votes agreed ({}) — no reconsideration",
7419                        initial_votes[0].label()
7420                    ),
7421                );
7422            }
7423
7424            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7425            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7426            // A round that already has a blocking finding and a round left to
7427            // try is going back to the fixer no matter what `verify.e2e`
7428            // says, so running it first only spends the loop's slowest step
7429            // (minutes, for a Rust repo's full test suite) on a head about
7430            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7431            // runs once a round has no blocking findings left (see
7432            // `round_is_clean`, which a deferred — empty — `e2e` can never
7433            // satisfy since `blocking` is nonzero whenever this branch is
7434            // taken), and `stop_reviewing` forces a real run before it will
7435            // ever read a deferred round as green.
7436            let defer_e2e =
7437                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7438            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7439                let reason =
7440                    format!("{blocking} blocking finding(s) already required a fix this round");
7441                self.state.event(
7442                    "verify",
7443                    format!(
7444                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7445                         {}); it will run once a round has none left",
7446                        short(&head)
7447                    ),
7448                );
7449                (Vec::new(), false, true, Some(reason))
7450            } else {
7451                let e2e_commands = self.state.config.verify.e2e.clone();
7452                let cache_dir = self.state.config.cache_dir();
7453                let context = format!("round {round}");
7454                let wait = self.repository_wait();
7455                let (e2e, verify_retried) = with_cache_lease(
7456                    &mut self.state,
7457                    cache_dir.as_deref(),
7458                    "e2e",
7459                    "e2e",
7460                    &winner.worktree,
7461                    &head,
7462                    verify_timeout,
7463                    &context,
7464                    &wait,
7465                    |state, budget| {
7466                        let shell = shell.clone();
7467                        let e2e_commands = e2e_commands.clone();
7468                        let worktree = winner.worktree.clone();
7469                        let context = context.clone();
7470                        async move {
7471                            run_e2e_with_retry(
7472                                state,
7473                                &shell,
7474                                &e2e_commands,
7475                                &worktree,
7476                                budget,
7477                                &context,
7478                            )
7479                            .await
7480                        }
7481                    },
7482                )
7483                .await?;
7484                (e2e, verify_retried, false, None)
7485            };
7486
7487            let expected = records.len();
7488            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7489            let incomplete = answered < expected;
7490            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7491            let policy = self.state.config.graph.incomplete_review;
7492            let clean = round_is_clean(
7493                blocking,
7494                e2e_ok,
7495                answered,
7496                expected,
7497                round_quota_missing,
7498                policy,
7499            );
7500
7501            let mut round_record = ReviewRound {
7502                round,
7503                head: head.clone(),
7504                verified_head: None,
7505                verified_at: None,
7506                reviews: records,
7507                e2e,
7508                verify_retried,
7509                e2e_deferred,
7510                e2e_defer_reason,
7511                fix: None,
7512                blocking,
7513                answered,
7514                expected,
7515                clean,
7516                progressed: false,
7517                vote_split,
7518                reconsideration,
7519                verdict: None,
7520            };
7521            // The final vote per seat is its revote where reconsideration
7522            // ran and answered, its initial vote otherwise — the same
7523            // fallback `tally` uses for a judge whose private vote failed.
7524            round_record.verdict = ReviewVote::worst(
7525                round_record
7526                    .final_votes()
7527                    .into_iter()
7528                    .map(|(_, _, vote)| vote),
7529            );
7530            // Which commit and when magi actually attempted to check —
7531            // known the moment a command was dispatched against `head`,
7532            // whether or not it finished: a resource-blocked attempt still
7533            // targeted a specific commit at a specific time, and leaving
7534            // that unrecorded is exactly what made `verification_summary`
7535            // report a fresh attempt as "commit unknown ... recorded before
7536            // this was tracked", indistinguishable from a genuinely old,
7537            // untracked record. Only a deferred or unconfigured round never
7538            // ran at all and has nothing to record — see
7539            // `ReviewRound::verified_head`'s own doc.
7540            if !matches!(
7541                round_record.e2e_status(),
7542                E2eStatus::Deferred | E2eStatus::NotConfigured
7543            ) {
7544                round_record.verified_head = Some(head.clone());
7545                round_record.verified_at = Some(Timestamp::now());
7546            }
7547
7548            if incomplete {
7549                let missing: Vec<String> = round_record
7550                    .reviews
7551                    .iter()
7552                    .filter(|r| r.failed.is_some())
7553                    .map(|r| format!("review-{}", r.reviewer))
7554                    .collect();
7555                self.state.event(
7556                    "review",
7557                    format!(
7558                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7559                        missing.join(", ")
7560                    ),
7561                );
7562            }
7563
7564            if clean {
7565                self.state.event(
7566                    "review",
7567                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7568                        format!(
7569                            "round {round}: clean (warn policy, incomplete panel) — no \
7570                             blocking findings from the seats that answered, verification green"
7571                        )
7572                    } else if incomplete {
7573                        format!(
7574                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7575                             quorum) — no blocking findings from the seats that answered, \
7576                             verification green",
7577                            expected - answered
7578                        )
7579                    } else {
7580                        format!("round {round}: clean — no blocking findings, verification green")
7581                    },
7582                );
7583                self.state.reviews.push(round_record);
7584                self.state.status = RunStatus::Gating;
7585                self.state.save()?;
7586                return Ok(());
7587            }
7588
7589            // Nothing was raised and verification passed, but not every seat
7590            // answered and `round_is_clean` still refused to call it clean —
7591            // either a seat is missing for a reason other than its own quota
7592            // (a crash, a timeout, unparsable output — worth another try), or
7593            // every seat that could have answered lost its quota and nobody
7594            // is left to decide on: re-review rather than send the fixer
7595            // after a round with nothing to fix.
7596            if incomplete && blocking == 0 && e2e_ok {
7597                self.state.reviews.push(round_record);
7598                self.state.save()?;
7599                if round == max_rounds {
7600                    self.state.status = RunStatus::Blocked;
7601                    self.state.event(
7602                        "review",
7603                        format!(
7604                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7605                             refusing to call it clean",
7606                            expected - answered
7607                        ),
7608                    );
7609                    return Ok(());
7610                }
7611                continue;
7612            }
7613
7614            // Nothing for the fixer to act on (`blocking == 0`) and the only
7615            // reason this round is not clean is that magi itself never got
7616            // a command to run — the shared build cache, not the patch (see
7617            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7618            // the fixer would invite a change to appease contention that has
7619            // nothing to do with the diff, and would leave this attempt
7620            // sitting in the next round's prompt as if it were about an
7621            // earlier, superseded commit rather than what it actually is:
7622            // the same head, still waiting to be checked. Wait for it the
7623            // same way the final round's own contention is already handled,
7624            // whatever round this happens to be.
7625            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7626                self.state.reviews.push(round_record);
7627                return self
7628                    .stop_unresolved_review(
7629                        "the round's own verification could not run",
7630                        &shell,
7631                        &winner.worktree,
7632                    )
7633                    .await;
7634            }
7635
7636            if round == max_rounds {
7637                self.state.reviews.push(round_record);
7638                return self
7639                    .stop_reviewing(
7640                        &format!(
7641                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7642                        ),
7643                        &shell,
7644                        &winner.worktree,
7645                    )
7646                    .await;
7647            }
7648
7649            if !self.fix_review_round(&winner, round_record).await? {
7650                return Ok(());
7651            }
7652        }
7653        Ok(())
7654    }
7655
7656    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7657    /// False means admission or continuation policy stopped the review loop.
7658    async fn fix_review_round(
7659        &mut self,
7660        winner: &Candidate,
7661        mut round_record: ReviewRound,
7662    ) -> Result<bool> {
7663        let round = round_record.round;
7664        let shell = self.state.config.shell();
7665        let step = self.run_fix_round(winner, &mut round_record).await?;
7666        if matches!(step, FixStep::Resource) {
7667            self.stop_for_resource()?;
7668            return Ok(false);
7669        }
7670        if self.state.reviews.len() < round {
7671            self.state.reviews.push(round_record);
7672        } else {
7673            self.state.reviews[round - 1] = round_record;
7674        }
7675        self.state.save()?;
7676
7677        // The fixer's own report never came back this round, even after
7678        // `continue_fix_report`'s own budget was spent on it — not an
7679        // ordinary "no report" (dropped stream, quota, plain failure),
7680        // which already reads that way and is left to the existing round
7681        // budget. Stopping here, rather than opening another round, is
7682        // what keeps a next reviewer/fixer wave from ever being
7683        // dispatched onto `winner.worktree` while whatever the seat's
7684        // last call may still have running there is unaccounted for: no
7685        // process liveness check exists (and none is being added — see
7686        // AGENTS.md/this task's own scope), so the only way to honour
7687        // "nothing starts before a valid report returns" is to not start
7688        // anything further on this worktree from this run at all.
7689        if matches!(step, FixStep::Recorded { report_lost: true }) {
7690            self.stop_reviewing(
7691                "the fixer's adoption report never came back, even after resuming its \
7692                         own seat; refusing to start another round against the same worktree \
7693                         while that is unresolved",
7694                &shell,
7695                &winner.worktree,
7696            )
7697            .await?;
7698            return Ok(false);
7699        }
7700
7701        let streak = self
7702            .state
7703            .reviews
7704            .iter()
7705            .rev()
7706            .take_while(|r| !r.progressed)
7707            .count();
7708        if streak >= STAGNANT_LIMIT {
7709            self.stop_reviewing(
7710                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7711                &shell,
7712                &winner.worktree,
7713            )
7714            .await?;
7715            return Ok(false);
7716        }
7717        Ok(true)
7718    }
7719
7720    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7721    async fn run_fix_round(
7722        &mut self,
7723        winner: &Candidate,
7724        round_record: &mut ReviewRound,
7725    ) -> Result<FixStep> {
7726        let round = round_record.round;
7727        let max_rounds = self.state.config.graph.review_rounds;
7728        let run_id = self.state.id.clone();
7729        let prompts = self.state.config.prompts.clone();
7730        let language = self.state.config.graph.language.clone();
7731        let sessions = self.state.config.graph.sessions;
7732        let artifacts = self.merge_repair_artifacts();
7733        let base = self.landing_base();
7734        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7735        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7736        let this_round_verification = round_record.verification_summary(&head);
7737        let all_findings: Vec<_> = round_record
7738            .reviews
7739            .iter()
7740            .flat_map(|r| r.findings.iter().cloned())
7741            .collect();
7742        // Fix. The winner's own implementer seat continues its conversation:
7743        // the competition is over, so context is pure benefit now.
7744        let blocking_findings: Vec<_> = all_findings
7745            .iter()
7746            .filter(|f| f.severity.blocks())
7747            .cloned()
7748            .collect();
7749        let fix_prompt = prompt::fix_in(
7750            &self.state.instruction,
7751            &blocking_findings,
7752            this_round_verification.as_ref(),
7753            round,
7754            max_rounds,
7755            &language,
7756            &self.state.config.graph.github_language,
7757        );
7758        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7759        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7760        let (job, seat, out) = self
7761            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7762                prompt: fix_prompt.clone(),
7763                spec,
7764                seat,
7765                cwd: winner.worktree.clone(),
7766                timeout,
7767                allow_write: true,
7768                sessions,
7769                artifacts: artifacts.clone(),
7770                stem: format!("fix-{round}"),
7771                handover: None,
7772            })
7773            .await;
7774        // Not asked: the round is not recorded (so a resume repeats it),
7775        // no fixer was handed over to, and the seat is left as it was.
7776        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7777            return Ok(FixStep::Resource);
7778        }
7779        let agent_id = seat.agent.clone();
7780
7781        let mut fix = FixRecord {
7782            agent: agent_id,
7783            addressed: Vec::new(),
7784            rejected: Vec::new(),
7785            notes: String::new(),
7786            committed: false,
7787            failed: None,
7788            duration_ms: 0,
7789            continuation: None,
7790        };
7791        let mut continuation = ContinuationRecord::not_needed();
7792        let mut final_seat = seat.clone();
7793        match out {
7794            AgentOutcome::Ok(o) => {
7795                fix.duration_ms = o.duration_ms;
7796                let parsed = verdict::extract_json::<FixReport>(&o.text);
7797                // A parsed report standing next to a command this same
7798                // reply's own CLI never confirmed the exit status of is
7799                // not a resolved answer — the identical `CommandEvidence`
7800                // `state.jobs` renders, read here instead of only on
7801                // display, per the completion judgment and the shown
7802                // record needing to agree.
7803                let incomplete_reason = match &parsed {
7804                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7805                        "the reply parsed, but it reported a command whose own CLI never \
7806                             confirmed an exit status"
7807                            .to_owned(),
7808                    ),
7809                    Ok(_) => None,
7810                    Err(e) => Some(e.to_string()),
7811                };
7812                match incomplete_reason {
7813                    None => {
7814                        let report = parsed.expect("checked Ok above");
7815                        fix.addressed = report.addressed;
7816                        fix.rejected = report.rejected;
7817                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7818                    }
7819                    Some(reason) => {
7820                        let (resumed_seat, resolved, failure, cont) = self
7821                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7822                            .await;
7823                        fix.duration_ms += cont.cumulative_wait_ms;
7824                        continuation = cont;
7825                        final_seat = resumed_seat;
7826                        match resolved {
7827                            Some(report) => {
7828                                fix.addressed = report.addressed;
7829                                fix.rejected = report.rejected;
7830                                fix.notes =
7831                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7832                            }
7833                            None => fix.failed = failure,
7834                        }
7835                    }
7836                }
7837            }
7838            // The CLI's raw error JSON is not a fix report to parse.
7839            AgentOutcome::Dropped(o) => {
7840                fix.duration_ms = o.duration_ms;
7841                let why = o
7842                    .dropped
7843                    .as_ref()
7844                    .map(|d| d.why.as_str())
7845                    .unwrap_or("the CLI ended the stream without delivering its answer");
7846                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7847            }
7848            AgentOutcome::Quota(o) => {
7849                self.state.quota.push(QuotaLoss::from_report(
7850                    final_seat.key.clone(),
7851                    "fix",
7852                    o.quota.as_ref(),
7853                ));
7854                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7855            }
7856            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7857        }
7858        if self.state.resource_block.is_some() {
7859            self.state.seats.insert(final_seat.key.clone(), final_seat);
7860            return Ok(FixStep::Resource);
7861        }
7862        fix.continuation = Some(continuation);
7863        self.state.seats.insert(final_seat.key.clone(), final_seat);
7864        let coordination = self.coordinate(&winner.worktree).await?;
7865        if let Ok(r) = git::rescue_commit(
7866            &winner.worktree,
7867            &format!("magi: review round {round} fixes (uncommitted work)"),
7868        )
7869        .await
7870        {
7871            self.state.note_withheld("fix", &r.withheld);
7872        }
7873        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7874        fix.committed = after != before;
7875        // Judged by what `git` says moved against base, never by the
7876        // fixer's own `addressed`/`rejected` count — see
7877        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7878        // `patch` snapshot above: swallowing this error would default
7879        // `diff_after` to empty, which almost always differs from a
7880        // non-empty `patch` and reads as "progressed" — exactly backwards
7881        // for a `git` failure the stagnation check cannot see through.
7882        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7883        drop(coordination);
7884        let progressed = diff_after != patch;
7885        let commit_note = if fix.committed {
7886            "committed"
7887        } else {
7888            "NO new commit"
7889        };
7890        let tree_note = if progressed {
7891            "changed vs base"
7892        } else {
7893            "unchanged vs base"
7894        };
7895        self.state.event(
7896            "fix",
7897            match &fix.failed {
7898                // Distinct on purpose from "0 addressed, 0 rejected": the
7899                // fixer's own diff still landed (blocking counts do keep
7900                // falling round over round), only its adoption report did
7901                // not come back, so this must never read like every
7902                // finding was reviewed and declined.
7903                Some(reason) => {
7904                    format!(
7905                        "round {round}: fixer's adoption report was lost ({reason}); \
7906                             {commit_note}, tree {tree_note}"
7907                    )
7908                }
7909                None => format!(
7910                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7911                         {tree_note}{}",
7912                    fix.addressed.len(),
7913                    fix.rejected.len(),
7914                    if continuation.outcome == ContinuationOutcome::Resumed {
7915                        format!(
7916                            " (adoption report recovered after {} continuation(s))",
7917                            continuation.attempts
7918                        )
7919                    } else {
7920                        String::new()
7921                    },
7922                ),
7923            },
7924        );
7925        round_record.fix = Some(fix);
7926        round_record.progressed = progressed;
7927        Ok(FixStep::Recorded {
7928            report_lost: matches!(
7929                continuation.outcome,
7930                ContinuationOutcome::Exhausted
7931                    | ContinuationOutcome::QuotaLost
7932                    | ContinuationOutcome::NoSession
7933            ),
7934        })
7935    }
7936
7937    /// Decide, from the last recorded round's own verification, whether
7938    /// stopping the review loop is a hand-off or a genuine block.
7939    ///
7940    /// Called once the loop has given up trying — the round budget is spent,
7941    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7942    /// findings still open, never while a round is still clean or the
7943    /// incomplete-panel case handled inline above. Gate and e2e are facts
7944    /// about the tree; a lingering review finding is an opinion, and this
7945    /// workload's own `magi stats` puts reviewer precision low enough
7946    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7947    /// must not by itself stand between a green, verified change and the
7948    /// human who decides what to do with it. A red e2e is not an opinion, so
7949    /// that case still blocks, with the failing command and a tail of its
7950    /// output recorded here rather than left in `run.json` for someone to go
7951    /// find.
7952    ///
7953    /// A round that deferred its own e2e (see [`Config::graph`]'s
7954    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7955    /// only because nothing ran, and treating an empty list as a passing one
7956    /// here is exactly the "deferred painted green" bug this function exists
7957    /// to not have. When the last round's own verification never resolved —
7958    /// deferred on purpose, or a real attempt the shared build cache blocked
7959    /// — this makes (or retries) the real run, on the actual worktree this
7960    /// loop is about to stop touching, before deciding anything. A
7961    /// resource-blocked attempt is likewise never read as either green or
7962    /// red: it is evidence about the machine, not the patch (see
7963    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7964    /// blocked cache leaves this call without deciding rather than guessing
7965    /// — the caller retries on a later reentry.
7966    /// Record, once, that the review loop handed off over a blocking finding
7967    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7968    /// `land` asks the owner even with `land_approval` off. Called from every
7969    /// path that concludes `Gating`; a reentry keeps the first record.
7970    fn record_contested_handoff(&mut self) {
7971        if self.state.contested_handoff.is_some() {
7972            return;
7973        }
7974        let Some(contested) = self
7975            .state
7976            .reviews
7977            .last()
7978            .and_then(ReviewRound::contested_handoff)
7979        else {
7980            return;
7981        };
7982        self.state.event(
7983            "review",
7984            format!(
7985                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7986                 wait for the owner's approval",
7987                contested.findings.len(),
7988                contested.rejecters.len()
7989            ),
7990        );
7991        self.state.contested_handoff = Some(contested);
7992    }
7993
7994    async fn stop_unresolved_review(
7995        &mut self,
7996        why: &str,
7997        shell: &[String],
7998        worktree: &Path,
7999    ) -> Result<()> {
8000        self.stop_reviewing_inner(why, shell, worktree, true).await
8001    }
8002
8003    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8004        self.stop_reviewing_inner(why, shell, worktree, false).await
8005    }
8006
8007    async fn stop_reviewing_inner(
8008        &mut self,
8009        why: &str,
8010        shell: &[String],
8011        worktree: &Path,
8012        repairable: bool,
8013    ) -> Result<()> {
8014        let round_idx = self.state.reviews.len() - 1;
8015        // A deferred round and a resource-blocked one are the same shape
8016        // here: neither has a real result yet, and both get one more
8017        // attempt. Read off `e2e_status` — the single source for this —
8018        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8019        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8020        // still retries instead of being read as a settled result the
8021        // instant it stops being empty.
8022        let needs_catchup_run = matches!(
8023            self.state.reviews[round_idx].e2e_status(),
8024            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8025        );
8026        if needs_catchup_run {
8027            let round = self.state.reviews[round_idx].round;
8028            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8029            let commands = self.state.config.verify.e2e.clone();
8030            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8031            let cache_dir = self.state.config.cache_dir();
8032            let context = format!(
8033                "round {round}: verification unresolved, catching up before the final decision"
8034            );
8035            let wait = self.repository_wait();
8036            let (outcomes, verify_retried) = with_cache_lease(
8037                &mut self.state,
8038                cache_dir.as_deref(),
8039                "e2e",
8040                "e2e",
8041                worktree,
8042                &attempted_head,
8043                timeout,
8044                &context,
8045                &wait,
8046                |state, budget| {
8047                    let shell = shell.to_vec();
8048                    let commands = commands.clone();
8049                    let context = context.clone();
8050                    async move {
8051                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8052                            .await
8053                    }
8054                },
8055            )
8056            .await?;
8057            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8058                self.state.record_base_progress(&attempted_head);
8059            }
8060            let last = &mut self.state.reviews[round_idx];
8061            last.e2e = outcomes;
8062            last.verify_retried = verify_retried;
8063            // Always the commit and time this attempt actually targeted,
8064            // whether or not it happens to equal the reviewed `head` and
8065            // whether or not a command finished — see
8066            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8067            // attempt is recorded too, so a later reader sees "attempted
8068            // again at T2" rather than silence.
8069            last.verified_head = Some(attempted_head);
8070            last.verified_at = Some(Timestamp::now());
8071            if verify_inconclusive(&last.e2e) {
8072                // Still not a real result: `e2e_deferred` is left exactly
8073                // as it was, so `needs_catchup_run` above reads
8074                // `ResourceBlocked` (via `e2e_status`, which checks
8075                // `resource_blocked` before `e2e_deferred`) and retries
8076                // again on the next reentry, rather than recording
8077                // contention as a red e2e and blocking the run on it.
8078                self.state.save()?;
8079                return Ok(());
8080            }
8081            last.e2e_deferred = false;
8082            // A complete panel may have been saved as unclean solely because
8083            // verification could not run. Refresh that verdict with the new
8084            // outcomes before a granted repair consumes the round's result.
8085            // Incomplete panels retain their original policy/quota decision.
8086            if last.answered == last.expected {
8087                last.clean = round_is_clean(
8088                    last.blocking,
8089                    last.e2e.iter().all(CommandOutcome::ok),
8090                    last.answered,
8091                    last.expected,
8092                    0,
8093                    self.state.config.graph.incomplete_review,
8094                );
8095            }
8096        }
8097        let last = &self.state.reviews[round_idx];
8098        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8099
8100        match last.e2e_status() {
8101            E2eStatus::Failed => {
8102                let red: Vec<String> = last
8103                    .e2e
8104                    .iter()
8105                    .filter(|o| !o.ok())
8106                    .map(|o| {
8107                        format!(
8108                            "`{}` -> {:?}\n{}",
8109                            o.command,
8110                            o.code,
8111                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8112                        )
8113                    })
8114                    .collect();
8115                let can_repair = last.blocking == 0 && last.fix.is_none();
8116                self.state.reviews[round_idx].clean = false;
8117                self.state
8118                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8119                self.state.status = RunStatus::Blocked;
8120                if repairable
8121                    && can_repair
8122                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8123                {
8124                    self.state.status = RunStatus::Reviewing;
8125                }
8126            }
8127            // `needs_catchup_run` above already retried once this call; if
8128            // it is still blocked, this is magi's own admission it could
8129            // not get a command to run, never a verdict on the patch — the
8130            // run is left exactly where a later reentry can retry again.
8131            E2eStatus::ResourceBlocked => {
8132                self.state.event(
8133                    "review",
8134                    format!(
8135                        "{why}; e2e could not run (resource unavailable); not \
8136                         deciding yet"
8137                    ),
8138                );
8139            }
8140            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8141                self.state.event(
8142                    "review",
8143                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8144                );
8145                self.record_contested_handoff();
8146                self.state.status = RunStatus::Gating;
8147            }
8148        }
8149        self.state.save()?;
8150        Ok(())
8151    }
8152
8153    // ----------------------------------------------------------------- gate
8154
8155    async fn gate(&mut self) -> Result<()> {
8156        if self.hold_base_sync().await? {
8157            return Ok(());
8158        }
8159        // Judged by the review record itself, not by `status`: a solo
8160        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8161        // reentry (see `judge`), and trusting it here is exactly how a run
8162        // that exhausted its review budget got gated and merged a second
8163        // time around. `review_conclusion` recomputes the review loop's own
8164        // verdict from the round records themselves — `Gating` for a clean
8165        // round or a hand-off (see `stop_reviewing`), anything else means the
8166        // loop is still going or genuinely blocked.
8167        // A base the winner could not be replayed onto is a decision, not a
8168        // round: there is no landing tree to gate. Read as its own record for
8169        // the same reason the review verdict is.
8170        if self.state.status == RunStatus::Failed
8171            || self
8172                .state
8173                .base_sync
8174                .as_ref()
8175                .is_some_and(|s| s.conflict.is_some())
8176            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8177                != Some(RunStatus::Gating)
8178        {
8179            return Ok(());
8180        }
8181        let Some(winner) = self.state.winner().cloned() else {
8182            return Ok(());
8183        };
8184        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8185        self.state.reconcile_verification_head(&head);
8186        self.state.save()?;
8187        let wait = self.repository_wait();
8188        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8189            return Ok(());
8190        }
8191        if self.state.gate_ran {
8192            // `review_loop` derives its conclusion from the clean review
8193            // record on every reentry and therefore puts a completed run back
8194            // in `Gating`. A recorded gate is a stronger, terminal fact:
8195            // retain its original command output (or lack of any, for a repo
8196            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8197            // doc) and restore `Blocked` on a real failure rather than
8198            // pretending the command is still running or running it a second
8199            // time. `gate_ran == false` remains the only shape — unattempted,
8200            // or a resource-blocked retry — that may still need to execute a
8201            // command.
8202            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8203                self.state.status = RunStatus::Blocked;
8204                self.state.save()?;
8205            }
8206            return Ok(());
8207        }
8208        self.state.status = RunStatus::Gating;
8209        let mut outcomes = self.run_gate(&winner).await?;
8210        let mut attempted_head = self.state.gate_head.clone();
8211        loop {
8212            // A resource-blocked outcome means the gate command never actually
8213            // ran - the shared build cache could not be acquired or confirmed
8214            // fresh in time - which is evidence about the machine, not about
8215            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8216            // Recording it as a red gate would mark a run `Blocked` on nothing
8217            // but contention magi has already logged; leaving `self.state.gate`
8218            // empty and `self.state.gate_ran` false instead keeps the shape
8219            // this function already treats as "still needs to run" (see the
8220            // early-return above), so the next call retries the command
8221            // rather than concluding anything.
8222            if verify_inconclusive(&outcomes) {
8223                self.state.save()?;
8224                return Ok(());
8225            }
8226            if outcomes.iter().all(CommandOutcome::ok) {
8227                break;
8228            }
8229            let fixes_before = self.state.gate_fixes.len();
8230            match self.gate_fix_round(&winner, &outcomes).await? {
8231                GateFix::Retry => {
8232                    outcomes = self.run_gate(&winner).await?;
8233                    attempted_head = self.state.gate_head.clone();
8234                }
8235                GateFix::Stop => {
8236                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8237                    if self.state.gate_fixes.len() > fixes_before
8238                        && attempted_head.as_deref() != Some(current.as_str())
8239                        && verify_current_head(&mut self.state, &winner, &wait).await?
8240                    {
8241                        // A new SHA owes a gate even when a spent fixer changed only
8242                        // commit metadata. This does not grant another repair round.
8243                        outcomes = self.run_gate(&winner).await?;
8244                        attempted_head = self.state.gate_head.clone();
8245                    } else {
8246                        break;
8247                    }
8248                }
8249                GateFix::Defer => {
8250                    self.state.save()?;
8251                    return Ok(());
8252                }
8253            }
8254        }
8255        let passed = outcomes.iter().all(CommandOutcome::ok);
8256        self.state.gate = outcomes;
8257        self.state.gate_ran = true;
8258        self.state.gate_head = attempted_head;
8259        if !self.state.gate.is_empty()
8260            && !verify_inconclusive(&self.state.gate)
8261            && let Some(head) = self.state.gate_head.clone()
8262        {
8263            self.state.record_base_progress(&head);
8264        }
8265        self.state.save()?;
8266        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8267            return Ok(());
8268        }
8269        if !passed {
8270            self.state.status = RunStatus::Blocked;
8271            let spent = self.state.gate_fixes.len();
8272            self.state.event(
8273                "gate",
8274                if spent == 0 {
8275                    "gate failed; not merging".to_owned()
8276                } else {
8277                    format!("gate failed after {spent} gate-fix round(s); not merging")
8278                },
8279            );
8280        }
8281        self.state.save()?;
8282        Ok(())
8283    }
8284
8285    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8286    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8287    /// after review is clean and never on a candidate awaiting judging.
8288    ///
8289    /// Never fails the run: a non-zero exit or timeout is a warning and a
8290    /// recorded outcome, and the gate remains the single arbiter. Nothing
8291    /// configured means nothing happens - no event, no commit. `commit_all`
8292    /// commits any leftover change under the neutral identity and returns
8293    /// `false` when the tree is clean, so no empty commit is ever made.
8294    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8295        let commands = state.config.verify.pre_gate.clone();
8296        if commands.is_empty() {
8297            return;
8298        }
8299        let shell = state.config.shell();
8300        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8301        let (outcomes, _) = run_commands(
8302            state,
8303            "pre_gate",
8304            "pre_gate",
8305            0,
8306            &shell,
8307            &commands,
8308            &winner.worktree,
8309            timeout,
8310        )
8311        .await;
8312        for o in &outcomes {
8313            if !o.ok() {
8314                tracing::warn!(
8315                    "pre_gate `{}` failed ({:?}); the gate decides",
8316                    o.command,
8317                    o.code
8318                );
8319            }
8320            state.event(
8321                "pre_gate",
8322                format!(
8323                    "`{}` -> {}",
8324                    o.command,
8325                    if o.ok() {
8326                        "pass".to_owned()
8327                    } else {
8328                        format!(
8329                            "FAIL ({:?})\n{}",
8330                            o.code,
8331                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8332                        )
8333                    }
8334                ),
8335            );
8336        }
8337        state.pre_gate = outcomes;
8338        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8339            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8340                Ok(head) => {
8341                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8342                    state.pre_gate_commit = Some(head);
8343                }
8344                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8345            },
8346            Ok(false) => {}
8347            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8348        }
8349        if let Err(e) = state.save() {
8350            tracing::warn!("could not persist the pre_gate record: {e:#}");
8351        }
8352    }
8353
8354    /// Run `verify.gate` once against the winner's current tree, logging one
8355    /// event per command. Empty when nothing is configured.
8356    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8357        let wait = self.repository_wait();
8358        run_landing_gate(&mut self.state, winner, &wait).await
8359    }
8360
8361    /// One bounded fix round for a failing gate.
8362    ///
8363    /// The fixer is told the failure came from the gate itself, not from a
8364    /// reviewer, and is shown the failed commands, their exit codes and a tail
8365    /// of their output - whatever `[verify].gate` holds, nothing here knows
8366    /// what those commands run. Only a normal non-zero exit that printed
8367    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8368    /// command or a full disk says nothing about the code, and a fixer sent
8369    /// after it can only appease the machine. The round is judged by what git
8370    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8371    /// again before the gate does, so a fix cannot trade a green gate for a
8372    /// red e2e unnoticed.
8373    async fn gate_fix_round(
8374        &mut self,
8375        winner: &Candidate,
8376        outcomes: &[CommandOutcome],
8377    ) -> Result<GateFix> {
8378        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8379            g.phase == crate::repair_budget::Phase::Fixing
8380                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8381        }) {
8382            self.state.gate_fixes.len().saturating_add(1)
8383        } else {
8384            self.state.config.graph.gate_fix_rounds
8385        };
8386        let spent = self.state.gate_fixes.len();
8387        if spent >= cap {
8388            if cap > 0 {
8389                self.state.event(
8390                    "gate",
8391                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8392                );
8393            }
8394            return Ok(GateFix::Stop);
8395        }
8396        if !gate_fixable(outcomes) {
8397            self.state.event(
8398                "gate",
8399                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8400                 command or similar); not spending a fix round on it",
8401            );
8402            return Ok(GateFix::Stop);
8403        }
8404        let min_free = self.state.config.disk.min_free_bytes;
8405        if min_free > 0 {
8406            match crate::disk::free_bytes(&winner.worktree) {
8407                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8408                Ok(free) => {
8409                    self.state.event(
8410                        "gate",
8411                        format!(
8412                            "only {free} bytes free ({min_free} required by `[disk] \
8413                             min_free_bytes`); not spending a fix round on a failure the disk \
8414                             may explain"
8415                        ),
8416                    );
8417                    return Ok(GateFix::Stop);
8418                }
8419                Err(e) => {
8420                    self.state.event(
8421                        "gate",
8422                        format!("free disk space could not be measured ({e:#}); no fix round"),
8423                    );
8424                    return Ok(GateFix::Stop);
8425                }
8426            }
8427        }
8428
8429        let attempt = spent + 1;
8430        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8431        let base = self.landing_base();
8432        let fix_prompt = prompt::gate_fix_in(
8433            &self.state.instruction,
8434            &failed,
8435            attempt,
8436            cap,
8437            &self.state.config.graph.language,
8438            &self.state.config.graph.github_language,
8439        );
8440        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8441        let sessions = self.state.config.graph.sessions;
8442        let artifacts = self.merge_repair_artifacts();
8443        self.state.event(
8444            "gate",
8445            format!("gate failed; gate-fix round {attempt} of {cap}"),
8446        );
8447        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8448        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8449        // Persist before invoking the fixer: it may commit and then be
8450        // interrupted before its result (or the following e2e) is saved.
8451        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8452            head: before.clone(),
8453            passed: false,
8454            outcomes: Vec::new(),
8455        });
8456        self.state.save()?;
8457        let (_, seat, out) = self
8458            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8459                prompt: fix_prompt.clone(),
8460                spec,
8461                seat,
8462                cwd: winner.worktree.clone(),
8463                timeout,
8464                allow_write: true,
8465                sessions,
8466                artifacts: artifacts.clone(),
8467                stem: format!("gate-fix-{attempt}"),
8468                handover: None,
8469            })
8470            .await;
8471        // The fixer was never asked: no round is spent (nothing is recorded)
8472        // and a later reentry asks the same agent again.
8473        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8474            return Ok(GateFix::Defer);
8475        }
8476        let mut record = GateFixRecord {
8477            agent: seat.agent.clone(),
8478            failed,
8479            notes: String::new(),
8480            committed: false,
8481            error: None,
8482        };
8483        match out {
8484            AgentOutcome::Ok(o) => {
8485                // A missing report is not a failed fix: the round is judged
8486                // by the tree below, and the report only carries prose.
8487                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8488                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8489                }
8490            }
8491            AgentOutcome::Dropped(_) => {
8492                record.error = Some("the CLI dropped the stream".to_owned());
8493            }
8494            AgentOutcome::Quota(o) => {
8495                self.state.quota.push(QuotaLoss::from_report(
8496                    seat.key.clone(),
8497                    "gate-fix",
8498                    o.quota.as_ref(),
8499                ));
8500                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8501            }
8502            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8503        }
8504        self.state.seats.insert(seat.key.clone(), seat);
8505        let coordination = self.coordinate(&winner.worktree).await?;
8506        if let Ok(r) = git::rescue_commit(
8507            &winner.worktree,
8508            &format!("magi: gate fix {attempt} (uncommitted work)"),
8509        )
8510        .await
8511        {
8512            self.state.note_withheld("gate-fix", &r.withheld);
8513        }
8514        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8515        record.committed = after != before;
8516        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8517        let note = record.error.clone();
8518        self.state.gate_fixes.push(record);
8519        self.state.save()?;
8520        if !changed {
8521            self.state.event(
8522                "gate",
8523                match note {
8524                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8525                    None => format!("gate-fix round {attempt}: the tree did not change"),
8526                },
8527            );
8528            return Ok(GateFix::Stop);
8529        }
8530        self.state.event(
8531            "gate",
8532            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8533        );
8534
8535        drop(coordination);
8536        let wait = self.repository_wait();
8537        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8538    }
8539
8540    // ---------------------------------------------------------------- merge
8541
8542    /// One read-only rewrite by the summary's author; a text the gate still
8543    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8544    /// applies on their word or on silence. `None` means the run is parked on
8545    /// that question and nothing may be pushed or opened yet.
8546    async fn guarded_pr_message(
8547        &mut self,
8548        winner: &Candidate,
8549        facts: Option<&BranchFacts>,
8550        posting: bool,
8551    ) -> Result<Option<PrMessage>> {
8552        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8553        let head = match final_head.as_deref() {
8554            Some(h) => crate::pr_outcome::FinalHead::At(h),
8555            None => crate::pr_outcome::FinalHead::Unknown,
8556        };
8557        // Only a pull request that is about to be posted pays for a summary.
8558        let summarized = if posting {
8559            Some(self.summarize_change(winner, final_head.as_deref()).await)
8560        } else {
8561            None
8562        };
8563        // A busy shared resource is not a permanent cause: publishing an
8564        // "unavailable" description would stick (a later head move never
8565        // edits the body). Stop before anything is pushed or opened; the
8566        // run is `Blocked` and a resume asks the summary again.
8567        if self.state.resource_block.is_some() {
8568            self.state.event(
8569                "pr-summary",
8570                "a shared resource was busy; nothing was published, resume to summarize again",
8571            );
8572            return Ok(None);
8573        }
8574        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8575            anyhow::bail!(
8576                "the final commit moved during public description generation; retry before publishing"
8577            );
8578        }
8579        let note = match (&summarized, &self.state.change_summary) {
8580            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8581            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8582            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8583            (None, _) => ChangeNote::NotAsked,
8584        };
8585        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8586        if !posting {
8587            let identity = crate::scrub::Identity::current();
8588            return Ok(Some(PrMessage {
8589                title: crate::scrub::scrub(&pr.title, &identity),
8590                body: crate::scrub::scrub(&pr.body, &identity),
8591            }));
8592        }
8593        // This very text was already rejected and asked about: no second
8594        // judge call, rewrite or question, just carry the record on.
8595        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8596        if let Some(known) = self
8597            .state
8598            .github_text
8599            .clone()
8600            .filter(|g| g.fingerprint == raw_fp)
8601        {
8602            return self.settle_withheld(winner, &pr, known).await;
8603        }
8604        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8605        let mut violations =
8606            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8607        let identity = crate::scrub::Identity::current();
8608        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8609            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8610            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8611        {
8612            violations.push(crate::github_text::Violation::SensitiveData);
8613        }
8614        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8615        // only a language violation starts one; sensitive data alone goes
8616        // straight to the owner, with the redacted text as the fallback.
8617        let has_language = violations
8618            .iter()
8619            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8620        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8621            self.state.event(
8622                "github-text",
8623                if has_language {
8624                    format!("description rejected: {violations:?}; requesting one rewrite")
8625                } else {
8626                    format!("description rejected: {violations:?}; asking the owner")
8627                },
8628            );
8629            let mut rewritten = false;
8630            let mut rejected_rewrite = None;
8631            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8632            {
8633                let output = text;
8634                if let Some(title) = summary_title(&output) {
8635                    // Judge the redacted candidate that will actually be posted.
8636                    // Sensitive data alone must not discard a useful rewrite.
8637                    let raw_body = summary_without_title(&output);
8638                    let clean_title = crate::scrub::scrub(&title, &identity);
8639                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8640                    let redacted = clean_title != title || body != raw_body;
8641                    let title = clean_title;
8642                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8643                    if !complete {
8644                        self.state.event(
8645                            "github-text",
8646                            "description rewrite omitted required findings",
8647                        );
8648                    }
8649                    if !body.trim().is_empty() {
8650                        body.push_str("\n\n");
8651                        body.push_str(&public_pr_evidence(&self.state, head));
8652                    }
8653                    let marker = format!("magi:run/{}", self.state.id);
8654                    if !body.trim().is_empty() && !body.contains(&marker) {
8655                        body.push_str(&format!("\n\n{marker}"));
8656                    }
8657                    let rewrite = PrMessage {
8658                        title: title.clone(),
8659                        body: body.clone(),
8660                    };
8661                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8662                    let mut failed = crate::github_text::check_for(
8663                        &self.state.config.graph,
8664                        &title,
8665                        &body,
8666                        redecision,
8667                    );
8668                    if (!crate::github_text::shareable(&title)
8669                        || !crate::github_text::shareable(&body))
8670                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8671                    {
8672                        failed.push(crate::github_text::Violation::SensitiveData);
8673                    }
8674                    if complete
8675                        && !title.trim().is_empty()
8676                        && !body.trim().is_empty()
8677                        && !failed.is_empty()
8678                    {
8679                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8680                            title: title.clone(),
8681                            body: body.clone(),
8682                            failed: failed
8683                                .iter()
8684                                .map(|v| crate::github_text::category(*v).to_owned())
8685                                .collect(),
8686                        });
8687                    }
8688                    if complete
8689                        && !title.trim().is_empty()
8690                        && !raw_body.trim().is_empty()
8691                        && failed.is_empty()
8692                    {
8693                        decision = redecision;
8694                        pr = PrMessage { title, body };
8695                        rewritten = true;
8696                        self.state
8697                            .event("github-text", "description rewrite passed");
8698                        if redacted {
8699                            self.state.event(
8700                                "github-text",
8701                                "sensitive data removed from description rewrite",
8702                            );
8703                        }
8704                    }
8705                }
8706            }
8707            if !rewritten {
8708                self.state.event(
8709                    "github-text",
8710                    "description rewrite unavailable or rejected; asking the owner",
8711                );
8712                if let Some(w) =
8713                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8714                {
8715                    // A different text than the one asked about before: that
8716                    // question no longer describes anything.
8717                    self.retire_github_text_question("the withheld text changed");
8718                    let record = crate::run::GithubTextAsk {
8719                        rewrite: rejected_rewrite,
8720                        fingerprint: raw_fp,
8721                        title: w.title,
8722                        body: w.body,
8723                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8724                        question: None,
8725                        asks: 0,
8726                        resolved: false,
8727                        chosen_title: None,
8728                        choice: None,
8729                    };
8730                    return self.settle_withheld(winner, &pr, record).await;
8731                }
8732            }
8733        }
8734        let (title, body) =
8735            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8736        Ok(Some(PrMessage { title, body }))
8737    }
8738
8739    /// Describe the whole change from its final diff in either run mode.
8740    ///
8741    /// The summary is tied to the final commit and to the merge base of the
8742    /// resolved remote base; a record for any other pair is dropped and the
8743    /// agent asked again, so a description never outlives the diff it was
8744    /// written from. The seat is read-only and the prompt tells it to read the
8745    /// diff and the code around it. `[roles] synthesizer` selects the agents
8746    /// (unset: the default pick), each asked at most once, advancing on an
8747    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8748    /// carries the reason it is unavailable; nothing is invented then.
8749    async fn summarize_change(
8750        &mut self,
8751        winner: &Candidate,
8752        head: Option<&str>,
8753    ) -> std::result::Result<(), String> {
8754        let Some(head) = head else {
8755            return Err("the final commit could not be read".to_owned());
8756        };
8757        let remote = self.state.config.merge.remote.clone();
8758        let tracking = format!("{remote}/{}", self.state.base_branch);
8759        if !git::rev_exists(&winner.worktree, &tracking).await {
8760            return Err(format!("the remote base `{tracking}` could not be read"));
8761        }
8762        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8763            Ok(b) if !b.is_empty() => b,
8764            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8765        };
8766        let evidence = crate::pr_outcome::summary_evidence(
8767            &self.state,
8768            crate::pr_outcome::FinalHead::At(head),
8769        );
8770        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8771        if self
8772            .state
8773            .change_summary
8774            .as_ref()
8775            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8776        {
8777            return Ok(());
8778        }
8779        if self.state.change_summary.take().is_some() {
8780            self.state.event(
8781                "pr-summary",
8782                "the final diff or evidence changed; the earlier change summary was discarded",
8783            );
8784        }
8785        // Save the source snapshot even if no generation agent is available.
8786        if let Err(e) = write_artifact(
8787            &self.state,
8788            &format!("public-description-{fingerprint}.md"),
8789            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8790        ) {
8791            tracing::warn!("could not save public description evidence: {e:#}");
8792        }
8793        let chain = match agent::pick_chain(
8794            &self.state.config.agents,
8795            self.state.config.roles.synthesizer.as_ref(),
8796            &agent::installed,
8797            "synthesizer",
8798        ) {
8799            Ok(chain) => chain,
8800            Err(e) => {
8801                tracing::warn!("no agent for the change summary: {e:#}");
8802                self.state.event("pr-summary", "no summary agent available");
8803                return Err("no agent was available to summarize it".to_owned());
8804            }
8805        };
8806        let subjects = git::subjects(&winner.worktree, &base, head)
8807            .await
8808            .unwrap_or_default();
8809        let stat = git::diff_stat(&winner.worktree, &base, head)
8810            .await
8811            .unwrap_or_default();
8812        let stat: String = stat.chars().take(6_000).collect();
8813        let mut prompt = crate::prompt::change_summary_in(
8814            &base,
8815            head,
8816            &subjects,
8817            stat.trim(),
8818            &self.state.config.graph.github_language,
8819        );
8820        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8821        let sessions = self.state.config.graph.sessions;
8822        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8823        let prompts = self.state.config.prompts.clone();
8824        let cache = self.state.config.cache_dir();
8825        let run = self.state.id.clone();
8826        for (i, spec) in chain.into_iter().enumerate() {
8827            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8828            let github_spec = match agent::for_github_text(
8829                &spec,
8830                &self.state.config.graph.github_language,
8831                &winner.worktree,
8832            ) {
8833                Ok(spec) => spec,
8834                Err(e) => {
8835                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8836                    continue;
8837                }
8838            };
8839            let job = SeatJob {
8840                spec: github_spec,
8841                seat,
8842                cwd: winner.worktree.clone(),
8843                prompt: prompt.clone(),
8844                timeout,
8845                allow_write: false,
8846                sessions,
8847                artifacts: agent::artifacts_dir(&self.state.dir()),
8848                stem: if i == 0 {
8849                    "change-summary".to_owned()
8850                } else {
8851                    format!("change-summary-{}", spec.id)
8852                },
8853                handover: None,
8854            };
8855            let ctx = WaveCtx {
8856                run: &run,
8857                node: "pr-summary",
8858                prompts: &prompts,
8859                cache: cache.as_deref(),
8860                round: None,
8861                carry_seats: false,
8862                pauses: Some((&self.pause, &self.interrupt)),
8863            };
8864            let (seat, outcome) =
8865                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8866            self.state.seats.insert(seat.key.clone(), seat);
8867            let text = match outcome {
8868                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8869                AgentOutcome::ResourceBlocked(why) => {
8870                    // No agent was asked: the run stops resumably before
8871                    // anything is published (see `guarded_pr_message`).
8872                    if self.state.resource_block.is_none() {
8873                        self.state.resource_block = Some(crate::run::ResourceBlock {
8874                            cause: crate::run::ResourceBlockCause::Other,
8875                            node: "pr-summary".to_owned(),
8876                            kind: "shared resource".to_owned(),
8877                            detail: why,
8878                            at: Timestamp::now(),
8879                        });
8880                    }
8881                    return Err("a shared resource was busy".to_owned());
8882                }
8883                _ => continue,
8884            };
8885            let blind = &self.state.config.blind;
8886            let Some(title) = summary_title(&text) else {
8887                continue;
8888            };
8889            let body: String = summary_without_title(&text)
8890                .lines()
8891                .filter(|l| !l.contains("magi:run/"))
8892                .collect::<Vec<_>>()
8893                .join("\n");
8894            let body = blind::sanitize_prose(body.trim(), blind);
8895            let title = blind::sanitize_prose(&title, blind);
8896            if body.trim().is_empty() || title.trim().is_empty() {
8897                continue;
8898            }
8899            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8900                continue;
8901            }
8902            // The branch must still be the one the agent was shown.
8903            if git::rev_parse(&winner.worktree, "HEAD")
8904                .await
8905                .ok()
8906                .as_deref()
8907                != Some(head)
8908            {
8909                return Err("the branch moved while the summary was written".to_owned());
8910            }
8911            if git::merge_base(&winner.worktree, &tracking, head)
8912                .await
8913                .ok()
8914                .as_deref()
8915                != Some(base.as_str())
8916                || crate::pr_outcome::summary_evidence(
8917                    &self.state,
8918                    crate::pr_outcome::FinalHead::At(head),
8919                ) != evidence
8920            {
8921                return Err("the summary evidence changed while it was written".to_owned());
8922            }
8923            self.state.change_summary = Some(crate::run::ChangeSummary {
8924                head: head.to_owned(),
8925                base,
8926                evidence: fingerprint,
8927                title,
8928                body,
8929            });
8930            self.state.event(
8931                "pr-summary",
8932                format!("change summary written by {} for the final diff", spec.id),
8933            );
8934            if let Err(e) = self.state.save() {
8935                tracing::warn!("could not save the change summary: {e:#}");
8936            }
8937            return Ok(());
8938        }
8939        self.state
8940            .event("pr-summary", "no agent produced a change summary");
8941        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8942    }
8943
8944    /// One read-only rewrite request for a withheld description. The
8945    /// implementer's own conversation is tried first when it exists and can be
8946    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8947    /// writes it in a separate conversation, skipping the implementer's agent.
8948    /// The prompt carries the whole text, so no agent needs any memory. Each
8949    /// agent is asked at most once and the chain advances on an error, quota,
8950    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8951    /// that has run out of quota must not end the rewrite. Nothing is recorded
8952    /// as a handover.
8953    async fn rewrite_pr_text(
8954        &mut self,
8955        winner: &Candidate,
8956        pr: &PrMessage,
8957        violations: &[crate::github_text::Violation],
8958    ) -> Option<String> {
8959        let prompt = format!(
8960            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8961            pr.title, pr.body
8962        );
8963        let sessions = self.state.config.graph.sessions;
8964        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8965        // (spec, the resumed implementer's seat). A fallback seat is created
8966        // only when its turn comes, so an unused fallback leaves no record.
8967        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8968        let key = format!("impl-{}", winner.label);
8969        if let Some(spec) = self
8970            .state
8971            .config
8972            .agents
8973            .iter()
8974            .find(|a| a.id == winner.agent)
8975            .cloned()
8976            && let Some(existing) = self.state.seats.get(&key)
8977            && existing.agent == spec.id
8978            && agent::has_session(spec.kind, existing, sessions)
8979        {
8980            attempts.push((spec, Some(existing.clone())));
8981        }
8982        match agent::pick_chain(
8983            &self.state.config.agents,
8984            self.state.config.roles.language_judge.as_ref(),
8985            &agent::installed,
8986            "language_judge",
8987        ) {
8988            Ok(chain) => {
8989                for spec in chain {
8990                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8991                        continue;
8992                    }
8993                    attempts.push((spec, None));
8994                }
8995            }
8996            Err(e) if attempts.is_empty() => {
8997                tracing::warn!("no agent for the description rewrite: {e:#}");
8998                self.state
8999                    .event("github-text", "no rewrite agent available");
9000                return None;
9001            }
9002            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9003        }
9004        let prompts = self.state.config.prompts.clone();
9005        let cache = self.state.config.cache_dir();
9006        let run = self.state.id.clone();
9007        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9008            let resumed = resumed_seat.is_some();
9009            let seat = resumed_seat.unwrap_or_else(|| {
9010                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9011            });
9012            let github_spec = match agent::for_github_text(
9013                &spec,
9014                &self.state.config.graph.github_language,
9015                &winner.worktree,
9016            ) {
9017                Ok(spec) => spec,
9018                Err(e) => {
9019                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9020                    continue;
9021                }
9022            };
9023            let job = SeatJob {
9024                spec: github_spec,
9025                seat,
9026                cwd: winner.worktree.clone(),
9027                prompt: prompt.clone(),
9028                timeout: retry_budget(timeout_full, resumed),
9029                allow_write: false,
9030                sessions,
9031                artifacts: agent::artifacts_dir(&self.state.dir()),
9032                stem: if i == 0 {
9033                    "github-text-rewrite".to_owned()
9034                } else {
9035                    format!("github-text-rewrite-{}", spec.id)
9036                },
9037                handover: None,
9038            };
9039            let ctx = WaveCtx {
9040                run: &run,
9041                node: "github-text",
9042                prompts: &prompts,
9043                cache: cache.as_deref(),
9044                round: None,
9045                carry_seats: false,
9046                pauses: Some((&self.pause, &self.interrupt)),
9047            };
9048            let (seat, outcome) =
9049                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9050            self.state.seats.insert(seat.key.clone(), seat);
9051            match outcome {
9052                AgentOutcome::Ok(o)
9053                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9054                {
9055                    return Some(o.text);
9056                }
9057                AgentOutcome::ResourceBlocked(_) => return None,
9058                _ => {}
9059            }
9060        }
9061        None
9062    }
9063
9064    /// Abandon the standing posting-gate question, if any.
9065    fn retire_github_text_question(&mut self, why: &str) {
9066        let Some(id) = self
9067            .state
9068            .github_text
9069            .as_ref()
9070            .and_then(|g| g.question.clone())
9071        else {
9072            return;
9073        };
9074        let _ = ask::Questions::open().update(&id, |q| {
9075            q.abandon(why);
9076            Ok(())
9077        });
9078    }
9079
9080    /// Carry a withheld text to its end: file the question, park while it is
9081    /// open, and on an answer, silence or timeout pick the text to post. The
9082    /// decision is saved before it is used, so it applies once.
9083    async fn settle_withheld(
9084        &mut self,
9085        winner: &Candidate,
9086        pr: &PrMessage,
9087        mut g: crate::run::GithubTextAsk,
9088    ) -> Result<Option<PrMessage>> {
9089        use crate::github_text as gt;
9090        let store = ask::Questions::open();
9091        let timeout = self.state.config.graph.answer_timeout;
9092        if !g.resolved {
9093            let mut standing = g
9094                .question
9095                .as_ref()
9096                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9097            if let Some(q) = standing.as_ref()
9098                && q.status.open()
9099                && gt::expired(q, timeout)
9100            {
9101                let why = format!("no answer within {}s of asking", timeout.max(1));
9102                standing = store
9103                    .update(&q.id, |q| {
9104                        q.abandon(&why);
9105                        Ok(())
9106                    })
9107                    .ok()
9108                    .map(|(q, ())| q);
9109            }
9110            let mut ask_again: Option<Option<Vec<String>>> = None;
9111            match standing {
9112                // Recorded but never written (a stop between the two saves),
9113                // or removed: ask again without spending the retry.
9114                None if g.asks <= 1 => {
9115                    g.asks = 0;
9116                    ask_again = Some(None);
9117                }
9118                None => {
9119                    g.resolved = true;
9120                    g.chosen_title = None;
9121                    g.choice = Some(GithubTextChoice::Fallback);
9122                }
9123                Some(q) if q.status.open() => {
9124                    self.park_on_github_text(&g)?;
9125                    return Ok(None);
9126                }
9127                Some(q) => match gt::read_reply(&q) {
9128                    gt::Reply::Fallback => {
9129                        g.resolved = true;
9130                        g.chosen_title = None;
9131                        g.choice = Some(GithubTextChoice::Fallback);
9132                    }
9133                    gt::Reply::Original => {
9134                        g.resolved = true;
9135                        g.chosen_title = None;
9136                        g.choice = Some(GithubTextChoice::Original);
9137                    }
9138                    gt::Reply::Rewrite => {
9139                        g.resolved = true;
9140                        g.chosen_title = None;
9141                        g.choice = Some(if g.rewrite.is_some() {
9142                            GithubTextChoice::Rewrite
9143                        } else {
9144                            GithubTextChoice::Fallback
9145                        });
9146                    }
9147                    gt::Reply::Title(text) => {
9148                        let first = text
9149                            .lines()
9150                            .map(str::trim)
9151                            .find(|l| !l.is_empty())
9152                            .unwrap_or("");
9153                        let decision = if first.is_empty() {
9154                            None
9155                        } else {
9156                            let candidate = PrMessage {
9157                                title: first.to_owned(),
9158                                body: String::new(),
9159                            };
9160                            self.judge_pr_language(&winner.worktree, &candidate).await
9161                        };
9162                        match gt::vet_title(
9163                            &text,
9164                            decision,
9165                            self.state.config.graph.github_english(),
9166                        ) {
9167                            Ok(title) => {
9168                                g.resolved = true;
9169                                g.chosen_title = Some(title);
9170                                g.choice = Some(GithubTextChoice::MyTitle);
9171                            }
9172                            Err(cats) if g.asks < 2 => {
9173                                ask_again = Some(Some(
9174                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9175                                ))
9176                            }
9177                            Err(_) => {
9178                                g.resolved = true;
9179                                g.chosen_title = None;
9180                                g.choice = Some(GithubTextChoice::Fallback);
9181                            }
9182                        }
9183                    }
9184                },
9185            }
9186            if let Some(rejected) = ask_again {
9187                // A retry (after a rejected replacement title) is still about
9188                // the original text: the same ORIGINAL / FALLBACK comparison.
9189                let w = gt::Withheld {
9190                    title: g.title,
9191                    body: g.body,
9192                    categories: g.categories.clone(),
9193                };
9194                // Named by the original text's fingerprint alone, so a resume
9195                // rewrites the same file and the detail stays identical.
9196                let artifact = match crate::run::write_artifact(
9197                    &self.state,
9198                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9199                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9200                ) {
9201                    Ok(p) => Some(p),
9202                    Err(e) => {
9203                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9204                        None
9205                    }
9206                };
9207                let (detail, comparison) = gt::question_content(
9208                    &w,
9209                    &pr.title,
9210                    &pr.body,
9211                    &self.state.id,
9212                    rejected.as_deref(),
9213                    artifact.as_deref(),
9214                    g.rewrite.as_ref(),
9215                );
9216                let mut q = ask::Question::new(
9217                    self.state.id.clone(),
9218                    gt::ASK_NODE.to_owned(),
9219                    gt::ASK_SEAT.to_owned(),
9220                    gt::question_summary(&self.state.config.graph.language, &w),
9221                    detail,
9222                    gt::question_choices(&w, g.rewrite.as_ref()),
9223                );
9224                q.posting_comparison = Some(comparison);
9225                q.answer_timeout = timeout;
9226                // Recorded before it is written: a stop in between leaves a
9227                // record naming a question the resume then files again.
9228                g.question = Some(q.id.clone());
9229                g.asks += 1;
9230                self.state.github_text = Some(g.clone());
9231                self.state.event(
9232                    "github-text",
9233                    format!("asking the owner about the withheld text ({})", q.short()),
9234                );
9235                self.state.save()?;
9236                store
9237                    .put_executable(&mut q)
9238                    .context("file the posting-gate question")?;
9239                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9240                    tracing::warn!(
9241                        "could not notify about posting-gate question {}: {e:#}",
9242                        q.short()
9243                    );
9244                }
9245                self.park_on_github_text(&g)?;
9246                return Ok(None);
9247            }
9248        }
9249        let resolved = g.chosen_title.clone();
9250        let original = g.choice == Some(GithubTextChoice::Original);
9251        // Field names only: the event log never records the text.
9252        self.state.event(
9253            "github-text",
9254            if original {
9255                format!(
9256                    "posting the original text for: {}",
9257                    match (g.title, g.body) {
9258                        (true, true) => "title, description",
9259                        (true, false) => "title",
9260                        _ => "description",
9261                    }
9262                )
9263            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9264                "posting the saved rewrite title and description".to_owned()
9265            } else if resolved.is_some() {
9266                "posting the owner's replacement title".to_owned()
9267            } else {
9268                "posting the neutral text for the withheld fields".to_owned()
9269            },
9270        );
9271        self.state.github_text = Some(g.clone());
9272        self.state.save()?;
9273        if g.choice == Some(GithubTextChoice::Rewrite)
9274            && let Some(r) = g.rewrite.as_ref()
9275        {
9276            return Ok(Some(PrMessage {
9277                title: r.title.clone(),
9278                body: r.body.clone(),
9279            }));
9280        }
9281        let w = gt::Withheld {
9282            title: g.title,
9283            body: g.body,
9284            categories: g.categories.clone(),
9285        };
9286        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9287        if let Some(t) = resolved {
9288            title = t;
9289        }
9290        // Every withheld field is now neutral or vetted English.
9291        let english = gt::LanguageDecision {
9292            title_english: true,
9293            body_english: true,
9294            body_complete: true,
9295        };
9296        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9297        if original {
9298            // The owner's explicit choice overrides the gate for the fields it
9299            // was made for; the others keep what the gate produced.
9300            if g.title {
9301                title = pr.title.clone();
9302            }
9303            if g.body {
9304                body = pr.body.clone();
9305            }
9306        }
9307        Ok(Some(PrMessage { title, body }))
9308    }
9309
9310    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9311        self.state.github_text = Some(g.clone());
9312        self.state.end_pass_parked(ParkCause::Question);
9313        self.state.parked = true;
9314        self.state.event(
9315            "github-text",
9316            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9317        );
9318        self.state.save()
9319    }
9320
9321    /// Ask `[roles] language_judge` about this text and record which source
9322    /// decided; the text itself is never recorded.
9323    async fn judge_pr_language(
9324        &mut self,
9325        cwd: &Path,
9326        pr: &PrMessage,
9327    ) -> Option<crate::github_text::LanguageDecision> {
9328        self.state.config.roles.language_judge.as_ref()?;
9329        if !self.state.config.graph.language_gate() {
9330            return None;
9331        }
9332        let decision =
9333            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9334        self.state.event(
9335            "github-text",
9336            if decision.is_some() {
9337                "language decided by the language judge"
9338            } else {
9339                "language judge unavailable; using built-in heuristics"
9340            },
9341        );
9342        decision
9343    }
9344
9345    async fn merge(&mut self) -> Result<()> {
9346        if self.resource_halt()? || self.hold_base_sync().await? {
9347            return Ok(());
9348        }
9349        // Same reasoning as `gate`: ask the review and gate records directly
9350        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9351        // skip can rewrite on reentry to something that no longer says
9352        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9353        // so a hand-off (open findings, green verification) reaches merge
9354        // exactly like a genuinely clean round does.
9355        //
9356        // A run resumed mid-`land` never reaches here at all: `execute`
9357        // recognises `RunStatus::Landing` before it even calls `prep`, and
9358        // routes straight to `run_land` instead. That has to happen a level
9359        // up from this function, not with a check in here, because
9360        // `review_loop`'s own status recomputation (see its doc) runs
9361        // *before* `merge` on every reentry and would otherwise overwrite
9362        // the `Landing` marker with `Gating` before this node ever saw it.
9363        if self
9364            .state
9365            .base_sync
9366            .as_ref()
9367            .is_some_and(|s| s.conflict.is_some())
9368            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9369                != Some(RunStatus::Gating)
9370            // `gate_ran == false` is not "passed" - `gate` leaves it false
9371            // both before it has ever run and when its last attempt was
9372            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9373            // permission to merge on nothing but the review record. Only a
9374            // gate that actually ran may proceed; an intentional local
9375            // review can also finish with verification explicitly not configured.
9376            // `RunState::gate_status` computes that distinction.
9377            || !self.state.gate_allows_progress()
9378        {
9379            return Ok(());
9380        }
9381        // This node's own record, not `status`: `status == Ready` is not
9382        // unique to the harmless `MergeMode::None` path this line was
9383        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9384        // run's PR was closed without merging — and on that run `mode` is
9385        // still `Pr`, so a reentry that fell through here would push and
9386        // open a second pull request. `self.state.merge` is set exactly once
9387        // this node (or `land`) has already produced a verdict, under every
9388        // mode, which is what "already done" actually means here.
9389        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9390            return Ok(());
9391        }
9392        let Some(winner) = self.state.winner().cloned() else {
9393            return Ok(());
9394        };
9395        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9396        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9397            self.state.status = RunStatus::Blocked;
9398            self.state.event(
9399                "gate",
9400                "winner changed after gate; fresh verification required",
9401            );
9402            self.state.save()?;
9403            return Ok(());
9404        }
9405        let wait = self.repository_wait();
9406        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9407            return Ok(());
9408        }
9409        let repo = self.state.repo.clone();
9410        let base = self.state.base_branch.clone();
9411        let mode = self.state.config.merge.mode;
9412        let style = self.state.config.merge.style;
9413        let facts = if is_review_run(&self.state) {
9414            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9415            let start = review_base(
9416                &repo,
9417                &self.state.config.merge.remote,
9418                &base,
9419                &self.state.base_commit,
9420                &winner.branch,
9421            )
9422            .await;
9423            branch_facts(&repo, &start, &winner.branch).await
9424        } else {
9425            None
9426        };
9427        // `None` when the base could not be freshly read: then an adopted
9428        // pull request's title is left alone.
9429        let leaked = if is_review_run(&self.state) {
9430            leaked_subjects(&self.state, &winner.branch).await
9431        } else {
9432            Some(Vec::new())
9433        };
9434        // `None`: parked on the owner's word about a withheld title or body.
9435        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9436        // the resume arrives here again.
9437        let Some(pr) = self
9438            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9439            .await?
9440        else {
9441            return Ok(());
9442        };
9443        let message = pr.commit_message();
9444
9445        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9446        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9447            return Ok(());
9448        }
9449        let outcome = match mode {
9450            MergeMode::None => MergeOutcome {
9451                mode,
9452                ok: true,
9453                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9454                empty: false,
9455            },
9456            MergeMode::Pr | MergeMode::Local
9457                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9458            {
9459                MergeOutcome {
9460                    mode,
9461                    ok: false,
9462                    detail: empty_candidate_detail(&self.state, &base),
9463                    empty: true,
9464                }
9465            }
9466            MergeMode::Local => {
9467                let on = git::current_branch(&repo).await?;
9468                if on.as_deref() != Some(base.as_str()) {
9469                    MergeOutcome {
9470                        mode,
9471                        ok: false,
9472                        detail: format!(
9473                            "{} has {} checked out, not the base branch {base}",
9474                            repo.display(),
9475                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9476                        ),
9477                        empty: false,
9478                    }
9479                } else if !git::is_clean(&repo).await? {
9480                    MergeOutcome {
9481                        mode,
9482                        ok: false,
9483                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9484                        empty: false,
9485                    }
9486                } else {
9487                    let out = match style {
9488                        MergeStyle::Merge => {
9489                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9490                        }
9491                        MergeStyle::Squash => {
9492                            git::merge_squash(&repo, &winner.branch, &message).await?
9493                        }
9494                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9495                    };
9496                    MergeOutcome {
9497                        mode,
9498                        ok: out.ok(),
9499                        detail: if out.ok() { out.stdout } else { out.stderr },
9500                        empty: false,
9501                    }
9502                }
9503            }
9504            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9505                self.state.pr_publication_pending = true;
9506                self.state.save()?;
9507                let out =
9508                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9509                self.state.pr_publication_pending = out.is_err();
9510                MergeOutcome {
9511                    mode,
9512                    ok: out.is_ok(),
9513                    detail: match out {
9514                        Ok(url) => url,
9515                        Err(e) => format!("{e:#}"),
9516                    },
9517                    empty: false,
9518                }
9519            }
9520            MergeMode::Pr => {
9521                let remote = self.state.config.merge.remote.clone();
9522                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9523                let pushed = if self.state.foreign_checkout.is_some() {
9524                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9525                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9526                } else {
9527                    git::push(&winner.worktree, &remote, &winner.branch).await?
9528                };
9529                if !pushed.ok() {
9530                    MergeOutcome {
9531                        mode,
9532                        ok: false,
9533                        detail: pushed.stderr,
9534                        empty: false,
9535                    }
9536                } else {
9537                    if self.state.foreign_checkout.is_some() {
9538                        self.state.pr_pushed_head = Some(delivery_head.clone());
9539                    }
9540                    // A retry or resume of a run whose branch already has an
9541                    // open pull request adopts it rather than failing on a
9542                    // duplicate. Only this winner branch into this base:
9543                    // `branch_for` derives the name from the run id, so a
9544                    // different run's pull request never matches.
9545                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9546                    let out = match pr_merge_plan(found) {
9547                        PrPlan::Create => {
9548                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9549                                .await
9550                        }
9551                        PrPlan::Adopt { url, title } => {
9552                            self.state
9553                                .event("merge", format!("Pr: adopted open pull request {url}"));
9554                            if self.state.github_text.as_ref().is_some_and(|g| {
9555                                g.resolved
9556                                    && g.choice == Some(GithubTextChoice::Rewrite)
9557                                    && g.rewrite.is_some()
9558                            }) {
9559                                // The owner chose the entire pair, even if the
9560                                // existing title already matches or was edited.
9561                                land::set_pr_text_verbatim(
9562                                    &winner.worktree,
9563                                    &url,
9564                                    &pr.title,
9565                                    &pr.body,
9566                                )
9567                                .await
9568                                .map(|()| url)
9569                            } else {
9570                                if title != pr.title
9571                                    && (!is_review_run(&self.state)
9572                                        || leaked
9573                                            .as_deref()
9574                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9575                                    && let Err(e) =
9576                                        if self.state.github_text.as_ref().is_some_and(|g| {
9577                                            g.resolved
9578                                                && g.title
9579                                                && g.choice == Some(GithubTextChoice::Original)
9580                                        }) {
9581                                            // The owner chose to post this title unmodified.
9582                                            land::set_pr_title_verbatim(
9583                                                &winner.worktree,
9584                                                &url,
9585                                                &pr.title,
9586                                            )
9587                                            .await
9588                                        } else {
9589                                            land::set_pr_title(
9590                                                &mut self.state,
9591                                                &winner.worktree,
9592                                                &url,
9593                                                &pr.title,
9594                                            )
9595                                            .await
9596                                        }
9597                                {
9598                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9599                                    self.state
9600                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9601                                }
9602                                Ok(url)
9603                            }
9604                        }
9605                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9606                    };
9607                    match out {
9608                        Ok(url) => MergeOutcome {
9609                            mode,
9610                            ok: true,
9611                            detail: url,
9612                            empty: false,
9613                        },
9614                        Err(e) => MergeOutcome {
9615                            mode,
9616                            ok: false,
9617                            detail: e.to_string(),
9618                            empty: false,
9619                        },
9620                    }
9621                }
9622            }
9623        };
9624
9625        self.state.status = match (mode, outcome.ok) {
9626            (MergeMode::None, _) => RunStatus::Ready,
9627            (_, true) if outcome.empty => RunStatus::Ready,
9628            (MergeMode::Pr, true) => RunStatus::Ready,
9629            (_, true) => RunStatus::Merged,
9630            (_, false) => RunStatus::Blocked,
9631        };
9632        self.state.event(
9633            "merge",
9634            format!(
9635                "{:?}: {}",
9636                mode,
9637                outcome.detail.lines().next().unwrap_or("")
9638            ),
9639        );
9640        self.state.merge = Some(outcome);
9641        self.state.save()?;
9642
9643        // The PR is open and the run would historically stop here, leaving the
9644        // operator to watch checks, feed review comments back to a fixer, and
9645        // merge. That was done by hand six times in one session before this
9646        // existed. Opt-in, because merging is the one irreversible thing magi
9647        // can do to a repository.
9648        if self.state.config.graph.land
9649            && mode == MergeMode::Pr
9650            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9651        {
9652            self.run_land().await?;
9653        }
9654        // `run_land` may have left `status` at `Landing` - still waiting on
9655        // CI or the owner's approval, not actually settled - so this has to
9656        // read whatever `status` ended up as here, not the `Merged` this
9657        // function set a few lines up.
9658        self.settle_questions();
9659        Ok(())
9660    }
9661
9662    /// Enter `land`.
9663    ///
9664    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9665    /// a resumed run's re-entry. `land::land` itself is what serialises the
9666    /// two git-mutating moments inside the loop — the rebase push and
9667    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9668    /// to hold a lock across the whole call, and doing so would serialise
9669    /// this run's CI wait against a *different* run's land-approval resume
9670    /// in the same repository, which is exactly the "must not wait on
9671    /// another task" property the daemon's slot-freeing exists to give.
9672    async fn run_land(&mut self) -> Result<()> {
9673        crate::pr_head::ensure_owner(&self.state)?;
9674        let url = self
9675            .state
9676            .pr
9677            .as_ref()
9678            .map(|pr| pr.url.clone())
9679            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9680            .unwrap_or_default();
9681        let url = url.lines().next().unwrap_or("").trim().to_owned();
9682        if !url.starts_with("http") {
9683            return Ok(());
9684        }
9685        // A land failure is not a lost run: the work is on a branch and the
9686        // pull request is open, which is exactly where a human takes over.
9687        loop {
9688            match land::land(&mut self.state, &url).await {
9689                Ok(_)
9690                    if self
9691                        .state
9692                        .merge_repairs
9693                        .last()
9694                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9695                {
9696                    if self.repair_merge_review().await? {
9697                        continue;
9698                    }
9699                }
9700                Ok(_) if self.state.resource_block.is_some() => {
9701                    return self.stop_for_resource();
9702                }
9703                Ok(pr) if self.state.parked => {
9704                    // `land` already saved the parked marker; nothing here
9705                    // overrides `status` back to a terminal value while an
9706                    // approval is still outstanding.
9707                    let _ = pr;
9708                }
9709                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9710                    self.state.status = RunStatus::Superseded;
9711                    self.state.save()?;
9712                }
9713                Ok(pr) => {
9714                    self.state.status = match pr.state {
9715                        land::PrLifecycle::Merged => RunStatus::Merged,
9716                        _ => RunStatus::Blocked,
9717                    };
9718                    // Downstream of a confirmed merge only - see
9719                    // `bump::should_release_bump`'s own doc for why this one
9720                    // check covers all three of `land`'s success paths.
9721                    // Best-effort: the run already landed, so a failure here
9722                    // (the decision call, `gh`, `cargo`) is recorded and never
9723                    // turns a landed run into a failed one.
9724                    if bump::should_release_bump(self.state.status)
9725                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9726                    {
9727                        // The event is the run's own record. Not-eligible cases
9728                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9729                        // `Err` is a bump that was tried and failed:
9730                        // `after_merge` itself raises the operator notice for
9731                        // that, whether or not a release PR exists yet.
9732                        self.state
9733                            .event("bump", format!("release bump skipped: {e:#}"));
9734                    }
9735                    // Independent of the bump, and best-effort in the same way:
9736                    // findings the merge left open become follow-up tasks.
9737                    if self.state.status == RunStatus::Merged {
9738                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9739                    }
9740                    self.state.save()?;
9741                }
9742                Err(e) => {
9743                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9744                        return Err(e);
9745                    }
9746                    self.state.status = RunStatus::Blocked;
9747                    self.state.event("land", format!("gave up: {e}"));
9748                    self.state.save()?;
9749                }
9750            }
9751            break;
9752        }
9753        Ok(())
9754    }
9755
9756    /// One repair cycle stays under the existing driver/task claim. An
9757    /// interrupted fixer is rolled back and held rather than run twice.
9758    async fn repair_merge_review(&mut self) -> Result<bool> {
9759        use crate::run::MergeRepairPhase as Phase;
9760        crate::pr_head::ensure_owner(&self.state)?;
9761        let i = self.state.merge_repairs.len() - 1;
9762        let request = self.state.merge_repairs[i].clone();
9763        if request.phase == Phase::Fixing {
9764            let restored = match self.state.winner().cloned() {
9765                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9766                None => Err(anyhow::anyhow!("repair has no winning branch")),
9767            };
9768            let why = match restored {
9769                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9770                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9771                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9772            };
9773            self.hold_merge_repair(i, why)?;
9774            return Ok(false);
9775        }
9776        // A completed repair gate is a publication checkpoint. On reentry,
9777        // reject external changes before the ordinary gate path can replace
9778        // that evidence by verifying a new head. Requested repairs still
9779        // start their own review generation below.
9780        if request.phase == Phase::Reviewing && self.state.gate_ran {
9781            let winner = self
9782                .state
9783                .winner()
9784                .context("repair has no winning branch")?;
9785            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9786            if self.state.gate_head.as_deref() != Some(head.as_str())
9787                || !git::is_clean(&winner.worktree).await?
9788            {
9789                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9790                return Ok(false);
9791            }
9792        }
9793        let failure = if i >= self.state.config.graph.land_rounds
9794            || self.state.config.graph.review_rounds == 0
9795        {
9796            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9797        } else {
9798            None
9799        };
9800        if let Some(why) = failure {
9801            self.hold_merge_repair(i, why)?;
9802            return Ok(false);
9803        }
9804        if request.phase == Phase::Requested {
9805            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9806                Ok(pr) => pr,
9807                Err(e) => {
9808                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9809                    return Ok(false);
9810                }
9811            };
9812            self.state.merge_repairs[i].phase = Phase::Fixing;
9813            self.state.land_approval = None;
9814            self.state.land_armed_head = None;
9815            self.state.save()?;
9816            // Serialize the exact records, including finding ids/details,
9817            // reconsideration reasons, final votes and verification output.
9818            let evidence = serde_json::to_string_pretty(&request)?;
9819            let reason = format!(
9820                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9821            );
9822            let budget = self.state.config.graph.land_rounds;
9823            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9824            match result {
9825                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9826                Ok(land::Fixed::Deferred) => {
9827                    // An unavailable fixer did not spend this repair request.
9828                    // Restore partial fallback work before retrying it, as for
9829                    // an interrupted repair, without closing the request.
9830                    let worktree = self
9831                        .state
9832                        .winner()
9833                        .context("repair has no winning branch")?
9834                        .worktree
9835                        .clone();
9836                    self.graft_restore(&worktree, &request.head).await?;
9837                    self.state.merge_repairs[i].phase = Phase::Requested;
9838                    self.stop_for_resource()?;
9839                    return Ok(false);
9840                }
9841                Ok(land::Fixed::Failed(why)) => {
9842                    self.hold_merge_repair(
9843                        i,
9844                        format!(
9845                            "repair failed: {why}; inspect the branch and run an explicit review"
9846                        ),
9847                    )?;
9848                    return Ok(false);
9849                }
9850                Err(e) => {
9851                    self.hold_merge_repair(
9852                        i,
9853                        format!(
9854                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9855                        ),
9856                    )?;
9857                    return Ok(false);
9858                }
9859            }
9860            // Even no change gets a real new reviewer wave. The previous
9861            // evidence remains in the request; a former clean/concluded round
9862            // cannot short-circuit the new review or gate.
9863            self.begin_merge_repair_review(i)?;
9864        }
9865        self.review_loop().await?;
9866        self.gate().await?;
9867        if self.resource_halt()? || self.state.parked {
9868            return Ok(false);
9869        }
9870        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9871            != Some(RunStatus::Gating)
9872            || !self.state.gate_allows_progress()
9873        {
9874            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9875            return Ok(false);
9876        }
9877        // Review and gate may have added commits after the initial repair.
9878        // Publish exactly their final head to the existing PR, with a lease.
9879        let winner = self
9880            .state
9881            .winner()
9882            .cloned()
9883            .context("repair has no winning branch")?;
9884        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9885        if self.state.gate_head.as_deref() != Some(after.as_str())
9886            || !git::is_clean(&winner.worktree).await?
9887        {
9888            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9889            return Ok(false);
9890        }
9891        let expected = self
9892            .state
9893            .pr_pushed_head
9894            .as_deref()
9895            .unwrap_or(&request.head)
9896            .to_owned();
9897        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9898        // Push the checked commit itself: a local ref moving after the check
9899        // must not change which commit the forge receives.
9900        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9901            return Ok(false);
9902        }
9903        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9904        let refspec = format!("{after}:refs/heads/{public}");
9905        let push = git::git_raw(
9906            &winner.worktree,
9907            &["push", &lease, &self.state.config.merge.remote, &refspec],
9908        )
9909        .await;
9910        match push {
9911            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9912            Ok(out) => {
9913                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9914                return Ok(false);
9915            }
9916            Err(e) => {
9917                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9918                return Ok(false);
9919            }
9920        }
9921        self.state.merge_repairs[i].phase = Phase::Complete;
9922        self.state.status = RunStatus::Landing;
9923        self.state.event(
9924            "repair",
9925            "new review and gate completed; fresh owner merge decision required",
9926        );
9927        self.state.save()?;
9928        Ok(true)
9929    }
9930
9931    fn merge_repair_artifacts(&self) -> PathBuf {
9932        let artifacts = agent::artifacts_dir(&self.state.dir());
9933        match self
9934            .state
9935            .merge_repairs
9936            .last()
9937            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9938        {
9939            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9940            None => artifacts,
9941        }
9942    }
9943
9944    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9945        if let Some(e2e) = self.state.head_e2e.take() {
9946            self.state
9947                .verification_history
9948                .push(crate::run::VerificationEvidence {
9949                    superseded_by: None,
9950                    archived_at: Some(Timestamp::now()),
9951                    fixes_spent: Some(self.state.gate_fixes.len()),
9952                    gate_head: None,
9953                    gate_ran: false,
9954                    gate: Vec::new(),
9955                    e2e: Some(e2e),
9956                });
9957        }
9958        self.state.reviews.clear();
9959        self.state.gate.clear();
9960        self.state.gate_ran = false;
9961        self.state.gate_head = None;
9962        self.state.gate_fixes.clear();
9963        self.state.gate_repair_e2e = None;
9964        self.state.pre_gate.clear();
9965        self.state.pre_gate_commit = None;
9966        self.state.contested_handoff = None;
9967        self.state.land_reruns = None;
9968        self.state.land_approval = None;
9969        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9970        self.state.status = RunStatus::Reviewing;
9971        self.state.save()
9972    }
9973
9974    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9975        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9976        self.state.merge_repairs[index].failure = Some(why.clone());
9977        self.state.status = RunStatus::Blocked;
9978        self.state.event("repair", why);
9979        self.state.save()
9980    }
9981
9982    // -------------------------------------------------------------- helpers
9983
9984    /// Fetch or create a seat, keeping its conversation across nodes.
9985    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9986        if let Some(existing) = self.state.seats.get(key)
9987            && existing.agent == agent
9988        {
9989            return existing.clone();
9990        }
9991        // A seat that changes agent mints its session id from the agent too,
9992        // like a handover: the old agent's uuid is already taken by the CLI.
9993        let fresh = if self.state.seats.contains_key(key) {
9994            handover_seat(key, agent, self.state.next_seat_seed())
9995        } else {
9996            SeatState::new(key, agent, self.state.seed)
9997        };
9998        self.state.seats.insert(key.to_owned(), fresh.clone());
9999        fresh
10000    }
10001
10002    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10003    /// seat to another roster agent, in which case that agent. Nodes that
10004    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10005    /// reconsideration) must keep talking to whoever answered it, not slip
10006    /// back to the agent that failed it.
10007    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10008        match self.state.seats.get(key) {
10009            Some(s) if s.agent != spec.id => {
10010                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10011            }
10012            _ => spec,
10013        }
10014    }
10015
10016    /// A candidate rendered for judging, with the leak policy applied.
10017    fn view(&self, c: &Candidate) -> CandidateView {
10018        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10019            .unwrap_or_default();
10020        let (patch, _) = blind::sanitize_patch(
10021            &format!("candidate {} patch", c.label),
10022            &raw,
10023            &self.state.config.blind,
10024        );
10025        CandidateView {
10026            label: c.label,
10027            branch: c.branch.clone(),
10028            summary: c.summary.clone(),
10029            stat: c.stat.clone(),
10030            patch,
10031        }
10032    }
10033
10034    /// The full candidate set as prompt text, for seats with no live session.
10035    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10036        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10037        prompt::judge(
10038            "(see above)",
10039            &views,
10040            self.roles.judges.len(),
10041            base_short,
10042            "en",
10043        )
10044    }
10045
10046    /// The final-vote prompt with everything a seat that has no session of its
10047    /// own needs: the candidates, the seat's own ranking and reasons, and the
10048    /// anonymised deliberation it took part in (only when there was one, so a
10049    /// handed-over seat never sees more than the seat it replaces did). The
10050    /// `Final vote` heading stays first.
10051    fn vote_prompt_full(
10052        &self,
10053        j: usize,
10054        labels: &[char],
10055        language: &str,
10056        candidates: &[Candidate],
10057        base_short: &str,
10058    ) -> String {
10059        let mut text = format!(
10060            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10061            prompt::final_vote(labels, language),
10062            self.state.instruction,
10063            self.candidate_block(candidates, base_short)
10064        );
10065        if let Some(own) = self
10066            .state
10067            .judgements
10068            .get(j)
10069            .filter(|r| !r.ranking.is_empty())
10070        {
10071            let reasons = own
10072                .reasons
10073                .iter()
10074                .map(|(k, v)| format!("- {k}: {v}"))
10075                .collect::<Vec<_>>()
10076                .join("\n");
10077            text.push_str(&format!(
10078                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10079                own.ranking.iter().collect::<String>(),
10080                if reasons.is_empty() {
10081                    ""
10082                } else {
10083                    ", because:\n"
10084                }
10085            ));
10086        }
10087        if !self.state.deliberation.is_empty() {
10088            text.push_str("\n# What was argued before this vote\n");
10089            for t in self.transcript(&[], j) {
10090                text.push_str(&format!(
10091                    "\n## {}{}\n\n{}\n",
10092                    t.who,
10093                    if t.is_self { " (you)" } else { "" },
10094                    t.body.trim()
10095                ));
10096            }
10097        }
10098        text
10099    }
10100
10101    /// Anonymised transcript for judge `self_idx`.
10102    ///
10103    /// The initial rankings are always the opening statements. Seeding them
10104    /// only when no turn had been taken yet meant every judge after the first
10105    /// argued against a single voice instead of against the actual split — the
10106    /// disagreement is the information, so it is always on the table.
10107    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10108        let mut turns = Vec::new();
10109        for j in &self.state.judgements {
10110            if j.ranking.is_empty() {
10111                continue;
10112            }
10113            let reasons = j
10114                .reasons
10115                .iter()
10116                .map(|(k, v)| format!("- {k}: {v}"))
10117                .collect::<Vec<_>>()
10118                .join("\n");
10119            turns.push(Turn {
10120                who: format!("Judge {} (opening ranking)", j.judge),
10121                is_self: j.judge == self_idx + 1,
10122                body: format!(
10123                    "Ranked {}{}{reasons}",
10124                    j.ranking.iter().collect::<String>(),
10125                    if reasons.is_empty() {
10126                        ""
10127                    } else {
10128                        ", because:\n"
10129                    }
10130                ),
10131            });
10132        }
10133        for t in self
10134            .state
10135            .deliberation
10136            .iter()
10137            .flat_map(|r| r.turns.iter())
10138            .chain(current)
10139        {
10140            turns.push(Turn {
10141                who: format!("Judge {}", t.judge),
10142                is_self: t.judge == self_idx + 1,
10143                body: t.body.clone(),
10144            });
10145        }
10146        turns
10147    }
10148}
10149
10150/// Does this seat still hold the context a follow-up prompt would rely on?
10151fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10152    agent::has_session(spec.kind, seat, sessions)
10153}
10154
10155/// The next entry in `roster` after `start`, never wrapping back to the
10156/// front, whose id is not in `tried` yet.
10157///
10158/// Starts one past `start` rather than at the front of `roster`: `start` is
10159/// the seat's own original position, and a seat whose candidate slot already
10160/// sits on the roster's second entry must fall through to the third next, not
10161/// restart at the first — which is very likely a different candidate's own
10162/// agent already. Never wraps back past `start`, for the same reason: an
10163/// entry earlier in the roster than the seat's own position is almost
10164/// certainly some *other* candidate slot's own agent, and once the tail of
10165/// the roster is exhausted there are no more untried agents for *this* seat
10166/// to fall through to — the caller's fallback chain ends there, exactly as
10167/// "no further untried agents remain in the list for that seat" asks for.
10168///
10169/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10170/// the same id twice (an operator's `roles.implementers` typo, or a
10171/// `[[agents]]` list reused across roles) must not let
10172/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10173/// over `roster` either finds an untried id or runs out, so this always
10174/// terminates regardless of duplicates.
10175fn next_untried_in_roster<'a>(
10176    roster: &'a [AgentSpec],
10177    start: usize,
10178    tried: &BTreeSet<String>,
10179) -> Option<&'a AgentSpec> {
10180    roster
10181        .get(start + 1..)?
10182        .iter()
10183        .find(|s| !tried.contains(&s.id))
10184}
10185
10186/// The successor for a seat, shared by all four seat kinds (implement, judge,
10187/// review, advise). `others` holds the ids that *currently* occupy the other
10188/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10189/// sees them), so roster entries beyond the seat count act as spares: a failed
10190/// seat goes to an agent no other seat holds whenever the roster permits.
10191///
10192/// `carried` is `Some` only for the review loop's carried failure history.
10193/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10194/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10195/// whole roster: untried and not another seat's occupant; (3) the plain walk
10196/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10197/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10198/// a better outcome than an empty seat, and it keeps the answer to "is there
10199/// a successor at all" exactly what it was before occupants were considered,
10200/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10201/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10202/// carried failure is a cheaper bet than doubling an agent on the panel, and
10203/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10204/// the same round are handled one at a time, so a seat later in the batch
10205/// sees an earlier one's new occupant but not yet its own freed agent.
10206fn pick_successor<'a>(
10207    roster: &'a [AgentSpec],
10208    start: usize,
10209    tried: &BTreeSet<String>,
10210    carried: Option<&BTreeSet<String>>,
10211    others: &BTreeSet<String>,
10212) -> Option<&'a AgentSpec> {
10213    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10214    roster
10215        .get(start + 1..)
10216        .and_then(|tail| {
10217            tail.iter()
10218                .filter(free)
10219                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10220        })
10221        .or_else(|| {
10222            carried
10223                .is_some()
10224                .then(|| roster.iter().find(free))
10225                .flatten()
10226        })
10227        .or_else(|| match carried {
10228            Some(c) => next_for_seat(roster, start, tried, c),
10229            None => next_untried_in_roster(roster, start, tried),
10230        })
10231}
10232
10233/// The next agent for a seat that carries its failure history across rounds
10234/// (the review loop). `round_tried` is this round's own bound and starts
10235/// empty every round; `carried_failed` only decides priority.
10236///
10237/// First: an id walking forward from `start`, never wrapping, that is neither
10238/// tried this round nor failed in an earlier one. Only when that is exhausted
10239/// does it rescue: the first roster id (in roster order, so this one *does*
10240/// look before `start`) not yet tried this round, which is by then a carried
10241/// failure. Each id is rescued at most once per round, so it cannot loop.
10242fn next_for_seat<'a>(
10243    roster: &'a [AgentSpec],
10244    start: usize,
10245    round_tried: &BTreeSet<String>,
10246    carried_failed: &BTreeSet<String>,
10247) -> Option<&'a AgentSpec> {
10248    roster
10249        .get(start + 1..)?
10250        .iter()
10251        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10252        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10253}
10254
10255/// Where a reviewer seat starts a round: the agent that last answered it when
10256/// it is still on the roster and not marked failed, else the spec's own agent
10257/// unless it failed, else the next roster agent that has not failed, else the
10258/// spec's own agent again (the whole roster failed). Ids no longer on the
10259/// roster are ignored. An empty roster has no handover, so the spec stands.
10260fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10261    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10262        return spec;
10263    };
10264    let ok = |id: &str| !h.failed.contains(id);
10265    if let Some(last) = h.last_ok.as_deref()
10266        && ok(last)
10267        && let Some(s) = roster.iter().find(|s| s.id == last)
10268    {
10269        return s.clone();
10270    }
10271    if ok(&spec.id) {
10272        return spec;
10273    }
10274    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10275    roster
10276        .get(from + 1..)
10277        .into_iter()
10278        .flatten()
10279        .chain(roster.iter())
10280        .find(|s| ok(&s.id))
10281        .cloned()
10282        .unwrap_or(spec)
10283}
10284
10285/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10286/// seed so a CLI that mints its session id up front (`--session-id`) never
10287/// reuses the uuid the previous agent already opened under the same seat key.
10288pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10289    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10290}
10291
10292/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10293/// so the classifier below and the code that builds the message cannot drift.
10294const TIMED_OUT: &str = "timed out";
10295
10296impl FailClass {
10297    /// `None` for an answer; otherwise how the turn failed.
10298    fn of(out: &AgentOutcome) -> Option<Self> {
10299        match out {
10300            AgentOutcome::Ok(_) => None,
10301            AgentOutcome::Quota(_) => Some(Self::Quota),
10302            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10303            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10304            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10305            // Not a failure of the agent. Callers check for it first; `None`
10306            // here means "no class to hand over on".
10307            AgentOutcome::ResourceBlocked(_) => None,
10308        }
10309    }
10310
10311    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10312    fn stem_word(&self) -> &'static str {
10313        match self {
10314            Self::Quota => "quota",
10315            _ => "handover",
10316        }
10317    }
10318}
10319
10320/// The message's first line with its variable parts removed — digit runs and
10321/// path-like tokens — so "exit code 2" and "exit code 7" read
10322/// as one kind of failure.
10323fn failure_signature(msg: &str) -> String {
10324    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10325    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10326    // the historical machine key while the process diagnosis uses readable prose.
10327    if let Some(code) = line
10328        .strip_prefix("process exited with no usable output (exit code ")
10329        .and_then(|s| s.strip_suffix(')'))
10330        .and_then(|s| s.parse::<i32>().ok())
10331    {
10332        line = format!("exited with some({code}) and no usable output");
10333    } else if matches!(
10334        line.as_str(),
10335        "process exited with no usable output (no exit code reported)"
10336            | "process exited with no usable output (exit code unavailable)"
10337    ) {
10338        line = "exited with none and no usable output".to_owned();
10339    }
10340    let mut out = Vec::new();
10341    for word in line.split_whitespace() {
10342        if word.contains('/') || word.contains('\\') {
10343            out.push("<path>".to_owned());
10344            continue;
10345        }
10346        let mut w = String::new();
10347        let mut in_digits = false;
10348        for c in word.chars() {
10349            if c.is_ascii_digit() {
10350                if !in_digits {
10351                    w.push('#');
10352                }
10353                in_digits = true;
10354            } else {
10355                in_digits = false;
10356                w.push(c);
10357            }
10358        }
10359        out.push(w);
10360    }
10361    out.join(" ").chars().take(120).collect()
10362}
10363
10364/// Whether a seat that just failed with `cur` may go to the next roster agent.
10365/// A quota or a timeout always may. Any other failure may not when the agent
10366/// before it failed the same way: an error the prompt causes would otherwise
10367/// walk the whole roster. `prev` is the class of the immediately preceding
10368/// agent's failure, so a quota or timeout in between breaks the run of
10369/// identical failures by itself.
10370fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10371    match cur {
10372        FailClass::Quota | FailClass::Timeout => true,
10373        FailClass::Other(_) => prev != Some(cur),
10374    }
10375}
10376
10377/// A short human reason for a failed outcome, for the handover record.
10378fn fail_reason(out: &AgentOutcome) -> String {
10379    match out {
10380        AgentOutcome::Ok(_) => String::new(),
10381        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10382        AgentOutcome::Dropped(o) => format!(
10383            "the CLI dropped the stream ({})",
10384            o.dropped
10385                .as_ref()
10386                .map(|d| d.why.as_str())
10387                .unwrap_or("it ended without delivering its answer")
10388        ),
10389        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10390            e.lines().next().unwrap_or("").chars().take(160).collect()
10391        }
10392    }
10393}
10394
10395/// Note one handover in the run: the structured record and, in the timeline,
10396/// the sentence a person reads. A quota keeps the wording it always had.
10397pub(crate) fn record_handover(
10398    state: &mut RunState,
10399    node: &str,
10400    seat: &str,
10401    from: &str,
10402    to: &str,
10403    class: &FailClass,
10404    reason: &str,
10405) {
10406    let message = if *class == FailClass::Quota {
10407        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10408    } else {
10409        format!("{seat}: handed over {from} -> {to} ({reason})")
10410    };
10411    state.event(node, message);
10412    state.handovers.push(Handover {
10413        at: Timestamp::now(),
10414        node: node.to_owned(),
10415        seat: seat.to_owned(),
10416        from: from.to_owned(),
10417        to: to.to_owned(),
10418        reason: reason.to_owned(),
10419    });
10420}
10421
10422/// Did this reply report running a command whose own CLI never confirmed an
10423/// exit status?
10424///
10425/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10426/// command *finished* (see that type's own doc), so this can only be `true`
10427/// for a command whose completion event carried no readable exit code — not
10428/// for one that simply is not mentioned at all. That is the one signal this
10429/// crate can read, from the same record `state.jobs` renders, about a reply
10430/// standing next to work its own CLI cannot vouch for finishing; it is
10431/// deliberately not a check on the exit code's *value* (a fixer legitimately
10432/// runs a command that fails mid-iteration before it succeeds) and not a
10433/// guess at a command still running in the background (which emits no event
10434/// at all, and so leaves no evidence here to find).
10435fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10436    commands.iter().any(|c| c.exit_code.is_none())
10437}
10438
10439/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10440/// trusted as a verified no-op — the adoption guard's own text-level half.
10441///
10442/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10443/// timed out): a marker only earns the benefit of the doubt from a turn the
10444/// CLI itself vouches for finishing properly, the same house style
10445/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10446/// *fix* report to for `commands`. A candidate that timed out, exited
10447/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10448/// is, whatever prose it wrote — this returns `None` before it ever looks at
10449/// `text`. The remaining guards (the tree really is empty, the evidence is
10450/// non-empty) are the caller's: this only reads what the reply *claimed*.
10451fn verified_noop_claim(
10452    usable: bool,
10453    commands: &[agent::CommandEvidence],
10454    text: &str,
10455) -> Option<String> {
10456    (usable && !has_unconfirmed_command(commands))
10457        .then(|| verdict::verified_noop(text))
10458        .flatten()
10459}
10460
10461fn short(commit: &str) -> String {
10462    commit.chars().take(7).collect()
10463}
10464
10465fn make_executable(path: &Path) -> Result<()> {
10466    #[cfg(unix)]
10467    {
10468        use std::os::unix::fs::PermissionsExt as _;
10469        let mut perms = std::fs::metadata(path)?.permissions();
10470        perms.set_mode(0o755);
10471        std::fs::set_permissions(path, perms)?;
10472    }
10473    #[cfg(not(unix))]
10474    {
10475        let _ = path;
10476    }
10477    Ok(())
10478}
10479
10480/// What every seat in one batch shares: where the answers are attributed, the
10481/// prompt overlay they inherit, and the build cache they are told to use.
10482///
10483/// A struct rather than four more parameters: `wave` also needs the run's
10484/// state (to record who is answering right now) and the attempt number, and
10485/// eight positional arguments is both unreadable and a clippy error.
10486struct WaveCtx<'a> {
10487    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10488    /// paid for it.
10489    run: &'a str,
10490    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10491    node: &'a str,
10492    prompts: &'a Prompts,
10493    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10494    cache: Option<&'a Path>,
10495    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10496    /// `JobRecord::round`. `None` for every other node.
10497    round: Option<usize>,
10498    /// Carry each seat's failed-agent history across waves (the review loop
10499    /// only): start-of-round priority and handover choice read
10500    /// [`RunState::seat_history`], and every answer or failure writes it.
10501    carry_seats: bool,
10502    /// The run's park / interrupt requests, so a wait for a shared resource
10503    /// can be abandoned the moment the operator asks to park.
10504    pauses: Option<(&'a Pause, &'a Pause)>,
10505}
10506
10507/// Shared scheduling policy for operations and spawned seat invocations.
10508fn repository_wait(
10509    state: &RunState,
10510    pauses: Option<(&Pause, &Pause)>,
10511) -> crate::repo_lock::WaitPolicy {
10512    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10513    let inherited = crate::repo_lock::WaitPolicy::current();
10514    crate::repo_lock::WaitPolicy::new(
10515        Duration::from_secs(state.config.graph.timeout_resource_wait),
10516        move || {
10517            inherited.cancelled()
10518                || pauses
10519                    .as_ref()
10520                    .is_some_and(|(a, b)| a.parked() || b.parked())
10521        },
10522    )
10523}
10524
10525fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10526    if error.is::<crate::repo_lock::ResourceBlocked>() {
10527        state.resource_block = Some(crate::run::ResourceBlock {
10528            cause: crate::run::ResourceBlockCause::Other,
10529            node: "repository".into(),
10530            kind: if error.to_string().contains("disk pressure") {
10531                "disk pressure"
10532            } else {
10533                "repository coordination"
10534            }
10535            .into(),
10536            detail: error.to_string(),
10537            at: Timestamp::now(),
10538        });
10539        state.event("resource", error.to_string());
10540    }
10541}
10542
10543async fn coordinate_operation(
10544    state: &mut RunState,
10545    cwd: &Path,
10546    wait: &crate::repo_lock::WaitPolicy,
10547) -> Result<Option<crate::repo_lock::SeatLease>> {
10548    let result =
10549        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10550            .await;
10551    if let Err(e) = &result {
10552        record_repository_block(state, e);
10553    }
10554    result
10555}
10556
10557/// Run one job, honouring the parallelism budget.
10558async fn run_one(
10559    job: SeatJob,
10560    sem: Arc<Semaphore>,
10561    ctx: &WaveCtx<'_>,
10562    state: &mut RunState,
10563    attempt: usize,
10564) -> (SeatState, AgentOutcome) {
10565    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10566        .await
10567        .pop()
10568        .expect("one job in, one result out");
10569    (seat, out)
10570}
10571
10572/// Run every job concurrently, capped by the semaphore, preserving order.
10573///
10574/// Every seat in the batch is recorded into [`RunState::active`] before the
10575/// wave starts and cleared as each answer lands, so the run's own record says
10576/// who is still being waited on rather than only who finished.
10577async fn wave(
10578    jobs: Vec<SeatJob>,
10579    sem: Arc<Semaphore>,
10580    ctx: &WaveCtx<'_>,
10581    state: &mut RunState,
10582    attempt: usize,
10583) -> Vec<(usize, SeatState, AgentOutcome)> {
10584    let WaveCtx {
10585        run,
10586        node,
10587        prompts,
10588        cache,
10589        round,
10590        carry_seats: _,
10591        pauses,
10592    } = *ctx;
10593    for job in &jobs {
10594        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10595    }
10596    if let Err(e) = state.save() {
10597        // A failed persist of "who is answering right now" must not abort the
10598        // wave: the seats are already being asked, and the alternative is
10599        // losing the answers to save a status line nobody may even be
10600        // watching.
10601        tracing::warn!("could not persist in-progress seats: {e:#}");
10602    }
10603    // Hold the shared build cache's lease for the whole batch, not per job:
10604    // several candidates (an implement wave) or a fixer legitimately share
10605    // one cache concurrently within this run, and that stays untouched — a
10606    // single lease taken once for the whole wave and released once it is
10607    // done is what stops a *different* borrower (another run's own wave, its
10608    // e2e/gate, a human's `magi review`) from interleaving a build into the
10609    // same directory while this one is in flight. Best-effort, not
10610    // all-or-nothing: a wave that cannot get the lease within its own
10611    // longest job's budget still runs — an hour of paid implementer calls is
10612    // not thrown away over cache contention — but every write-allowed seat
10613    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10614    // below), the same fallback a read-only seat always gets, rather than
10615    // building into a directory this run was never granted. The identity
10616    // record is still invalidated below either way, so the next tracked
10617    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10618    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10619    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10620    // never borrowed from a job's agent budget (and never charged to it either,
10621    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10622    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10623    let cache_guard = if let Some(cache_dir) = cache {
10624        if jobs_had_a_writer {
10625            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10626            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10627            match acquire_cache_lease(
10628                state,
10629                cache_dir,
10630                &owner,
10631                resource_limit,
10632                node,
10633                node,
10634                &cancel,
10635            )
10636            .await
10637            {
10638                Ok(Some(g)) => Some(g),
10639                // Parking: leave without spawning anything. The run parks at
10640                // the next boundary and the seats are asked afresh on resume.
10641                Ok(None) => {
10642                    if let Some(block) = &mut state.resource_block {
10643                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10644                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10645                        };
10646                    }
10647                    return block_wave(
10648                        jobs,
10649                        state,
10650                        node,
10651                        attempt,
10652                        "parked while waiting for the shared build cache",
10653                    );
10654                }
10655                // An unreadable lease is not contention: keep the old
10656                // fallback (no shared cache for this wave's seats).
10657                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10658                Err(e) => {
10659                    // Writing seats may not build into a cache this run does
10660                    // not hold, and an agent that cannot be given one is not
10661                    // asked at all: the run stops resumably instead.
10662                    let why = format!("{e:#}");
10663                    state.event(
10664                        "resource",
10665                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10666                    );
10667                    return block_wave(jobs, state, node, attempt, &why);
10668                }
10669            }
10670        } else {
10671            None
10672        }
10673    } else {
10674        None
10675    };
10676    // Every job keeps its whole budget: the wait above was not the agent's.
10677    // Restart the seats' clocks so "started" means "being spawned now".
10678    state.seats_spawning(node, attempt);
10679    if let Err(e) = state.save() {
10680        tracing::warn!("could not persist the start of a wave: {e:#}");
10681    }
10682    // Queueing behind this wave's own seats for a parallelism slot is normal
10683    // (each of them is bounded by its own agent timeout); only a wait beyond
10684    // all of them plus the resource limit means the slots never came back.
10685    let slot_limit = resource_limit
10686        + jobs
10687            .iter()
10688            .map(|j| j.timeout)
10689            .max()
10690            .unwrap_or_default()
10691            .saturating_mul(jobs.len() as u32);
10692    let mut set = tokio::task::JoinSet::new();
10693    // Seats announce themselves here the moment they hold a parallelism
10694    // permit, i.e. just before their CLI is spawned.
10695    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10696    let overlay = prompts.overlay(node);
10697    let repository_wait = repository_wait(state, pauses);
10698    for (i, mut job) in jobs.into_iter().enumerate() {
10699        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10700        if cache.is_some() {
10701            job.prompt.push('\n');
10702            job.prompt
10703                .push_str(&prompt::build_cache_note(node, job.allow_write));
10704        }
10705        let sem = Arc::clone(&sem);
10706        let run = run.to_owned();
10707        let node = node.to_owned();
10708        // Only implementers were told about the task's attachments, so only
10709        // their seats get the directory widened for reading.
10710        let attachments = if node == "implement" {
10711            state.attachments.clone()
10712        } else {
10713            Vec::new()
10714        };
10715        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10716        // `prompt::build_cache_note`'s doc for why setting it anyway is
10717        // exactly how a sandboxed reviewer's write refusal got reported as a
10718        // defect in the patch, not a property of its own seat. And a
10719        // write-allowed one is handed it only when the lease above was
10720        // actually acquired: a wave that could not get it (`cache_guard` is
10721        // `None`, see its own comment) must not send seats to build into a
10722        // directory this run does not hold - that is the exact concurrent,
10723        // unmanaged-write race this module exists to prevent, not something
10724        // "proceeding anyway" is allowed to reintroduce.
10725        let cache = cache
10726            .filter(|_| job.allow_write && cache_guard.is_some())
10727            .map(Path::to_path_buf);
10728        let spawned_tx = spawned_tx.clone();
10729        let repository_wait = repository_wait.clone();
10730        let coordination = state.config.coordination.clone();
10731        let fetched_base = state.fetched_base.clone();
10732        set.spawn(async move {
10733            // A wait for a parallelism slot is scheduling, not the agent: it
10734            // is bounded by its own limit and ends as a resource block, never
10735            // as a timeout of the agent that was not started.
10736            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10737                Ok(p) => p,
10738                Err(_) => {
10739                    return (
10740                        i,
10741                        job.seat,
10742                        AgentOutcome::ResourceBlocked(format!(
10743                            "no parallel slot became free within {}s",
10744                            slot_limit.as_secs()
10745                        )),
10746                        false,
10747                    );
10748                }
10749            };
10750            spawned_tx.send(job.seat.key.clone()).ok();
10751            let mut seat = job.seat;
10752            let out = repository_wait
10753                .scope(agent::invoke(
10754                    &job.spec,
10755                    &mut seat,
10756                    &Invocation {
10757                        coordination: coordination.as_ref(),
10758                        fetched_base: fetched_base.as_ref(),
10759                        cwd: &job.cwd,
10760                        prompt: &job.prompt,
10761                        timeout: job.timeout,
10762                        allow_write: job.allow_write,
10763                        sessions: job.sessions,
10764                        artifacts: &job.artifacts,
10765                        stem: &job.stem,
10766                        run: &run,
10767                        node: &node,
10768                        cache_dir: cache.as_deref(),
10769                        attachments: &attachments,
10770                        writable: &[],
10771                    },
10772                ))
10773                .await;
10774            let interrupted = out
10775                .as_ref()
10776                .err()
10777                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10778            let out = match out {
10779                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10780                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10781                // Billed work the CLI failed to hand over is not an ordinary
10782                // failure, but its text is the CLI's raw error JSON, not an
10783                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10784                // read it as one by forgetting to check. `usable()` is always
10785                // false here (dropped implies an empty response), so this has
10786                // to be checked before the catch-all `Failed` below or the
10787                // one shape this exists for is lost with the rest.
10788                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10789                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10790                Ok(o) => AgentOutcome::Failed(format!(
10791                    "process exited with no usable output ({})",
10792                    crate::notices::exit_status(o.exit_code)
10793                )),
10794                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10795                    AgentOutcome::ResourceBlocked(e.to_string())
10796                }
10797                Err(e) => AgentOutcome::Failed(e.to_string()),
10798            };
10799            (i, seat, out, interrupted)
10800        });
10801    }
10802    drop(spawned_tx);
10803    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10804    loop {
10805        let joined = tokio::select! {
10806            Some(key) = spawned_rx.recv() => {
10807                // The clock of this seat starts now, not when it queued.
10808                state.seat_spawned(&key);
10809                if let Err(e) = state.save() {
10810                    tracing::warn!("could not persist a seat's start: {e:#}");
10811                }
10812                continue;
10813            }
10814            joined = set.join_next() => joined,
10815        };
10816        let Some(joined) = joined else {
10817            break;
10818        };
10819        let (i, seat, out, interrupted) = match joined {
10820            Ok(v) => v,
10821            // No seat to clear: a panicked task never reported which one it
10822            // was. The defensive sweep below this loop is what stops that
10823            // seat's `active` entry from surviving forever.
10824            Err(e) => {
10825                tracing::error!("agent task panicked: {e}");
10826                continue;
10827            }
10828        };
10829        state.seat_finished(&seat.key);
10830        if interrupted
10831            && node == "implement"
10832            && let Some(label) = state
10833                .candidates
10834                .iter()
10835                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10836                .map(|candidate| candidate.label)
10837        {
10838            state.candidate_preservation.insert(
10839                label,
10840                "interrupted implementation awaiting persistence".into(),
10841            );
10842            state.interrupted_implementations.insert(label);
10843        }
10844        if let AgentOutcome::ResourceBlocked(why) = &out
10845            && state.resource_block.is_none()
10846        {
10847            state.resource_block = Some(crate::run::ResourceBlock {
10848                cause: crate::run::ResourceBlockCause::Other,
10849                node: node.to_owned(),
10850                kind: if why.contains("disk pressure") {
10851                    "disk pressure"
10852                } else if why.contains("repository coordination") {
10853                    "repository coordination"
10854                } else {
10855                    "parallel slot"
10856                }
10857                .to_owned(),
10858                detail: why.clone(),
10859                at: Timestamp::now(),
10860            });
10861            let activity = if interrupted {
10862                "agent execution interrupted"
10863            } else {
10864                "no agent was started"
10865            };
10866            state.event("resource", format!("{node}: {activity} - {why}"));
10867        }
10868        record_jobs(state, node, round, &seat.key, &out);
10869        if let Err(e) = state.save() {
10870            tracing::warn!("could not persist a seat's completion: {e:#}");
10871        }
10872        if collected.len() <= i {
10873            collected.resize_with(i + 1, || None);
10874        }
10875        collected[i] = Some((i, seat, out));
10876    }
10877    // Belt-and-braces for the panic branch above: every seat this exact batch
10878    // started shares this `(node, attempt)` pair, and every seat that finished
10879    // normally already cleared itself, so anything left tagged with it here
10880    // can only be a panicked task's leftover. Cleared unconditionally rather
10881    // than left to read as still answering forever.
10882    if state
10883        .active
10884        .values()
10885        .any(|a| a.node == node && a.attempt == attempt)
10886    {
10887        state
10888            .active
10889            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10890        if let Err(e) = state.save() {
10891            tracing::warn!("could not persist the end of a wave: {e:#}");
10892        }
10893    }
10894    // Whether or not the lease above was actually held, several worktrees
10895    // may just have built into the cache with nothing here able to name one
10896    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10897    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10898    // clean it might not have strictly needed; trusting a stale match would
10899    // cost it a wrong answer.
10900    if let Some(cache_dir) = cache
10901        && jobs_had_a_writer
10902    {
10903        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10904    }
10905    if let Some(guard) = cache_guard {
10906        guard.release();
10907    }
10908    collected.into_iter().flatten().collect()
10909}
10910
10911/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10912/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10913/// this is data collection, not the fix-specific completion contract in
10914/// [`Runner::continue_fix_report`], and applies regardless of which node
10915/// asked.
10916///
10917/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10918/// evidence from; `Failed` does not, and correctly contributes nothing — a
10919/// timeout or crash is not itself evidence about a command the seat may have
10920/// started.
10921fn record_jobs(
10922    state: &mut RunState,
10923    node: &str,
10924    round: Option<usize>,
10925    seat: &str,
10926    out: &AgentOutcome,
10927) {
10928    let commands: &[agent::CommandEvidence] = match out {
10929        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10930        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10931    };
10932    let checked_at = Timestamp::now();
10933    for c in commands {
10934        state.jobs.push(JobRecord {
10935            node: node.to_owned(),
10936            round,
10937            seat: seat.to_owned(),
10938            id: c.id.clone(),
10939            description: c.description.clone(),
10940            checked_at,
10941            status: match c.exit_code {
10942                Some(0) => JobStatus::Completed,
10943                Some(_) => JobStatus::Failed,
10944                None => JobStatus::Unknown,
10945            },
10946            exit_code: c.exit_code,
10947            result_summary: c.result_summary.clone(),
10948            source: c.source.clone(),
10949        });
10950    }
10951}
10952
10953/// Is a review round clean, given how many reviewer seats answered against
10954/// how many the round expected?
10955///
10956/// A seat that never answered (timeout, crash, unparsable output) is not a
10957/// seat that read the patch and found nothing — treating it as such is
10958/// exactly the bug this function exists to close. Under the default `block`
10959/// policy a missing seat can never be clean; `warn` still requires the seats
10960/// that *did* answer to have found nothing blocking and verification to be
10961/// green.
10962///
10963/// `quota_missing` narrows that `block` default for exactly one cause of
10964/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
10965/// a session limit lifts by the very next round buys nothing — the seat is
10966/// asked again with the same quota — so once every missing seat is accounted
10967/// for by a quota loss (and at least one seat *did* answer, so a decision has
10968/// something to rest on) the round is decided on the panel that could answer,
10969/// same as `warn` would. A panel that lost every seat to quota is not
10970/// decided here: `answered == 0` falls through to the existing `block`
10971/// fallback so a fully collapsed panel still waits rather than landing on no
10972/// review at all.
10973fn round_is_clean(
10974    blocking: usize,
10975    e2e_ok: bool,
10976    answered: usize,
10977    expected: usize,
10978    quota_missing: usize,
10979    policy: IncompleteReviewPolicy,
10980) -> bool {
10981    if blocking != 0 || !e2e_ok {
10982        return false;
10983    }
10984    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10985        return true;
10986    }
10987    answered > 0 && expected - answered <= quota_missing
10988}
10989
10990/// The review loop's own conclusion, derived entirely from its persisted
10991/// round records and the round budget that produced them — never from
10992/// `status`, so a reentry (or `gate`/`merge` reading it independently)
10993/// recomputes the identical answer regardless of what an earlier node in the
10994/// same walk, or a previous walk, did to `status`.
10995///
10996/// `None` while more rounds remain to try, including when review never ran
10997/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
10998/// gone clean, or the budget is spent, or the tree has stopped moving (see
10999/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11000///
11001/// - An incomplete panel that raised nothing is missing input, not a
11002///   verified tree — never a hand-off candidate, whatever verification said
11003///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11004/// - Otherwise, green e2e on the last round hands off (see
11005///   [`Runner::stop_reviewing`]); red e2e blocks.
11006///
11007/// A last round whose own verification is still `ResourceBlocked` — magi
11008/// itself never got a command to run, not evidence the patch is broken —
11009/// is neither: this returns `None` for it too, the same as "more rounds
11010/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11011/// handling of that shape) instead of this cheap recomputation guessing a
11012/// verdict a real attempt never produced.
11013pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11014    if max_rounds == 0
11015        || reviews.last().is_some_and(|r| {
11016            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11017        })
11018    {
11019        return Some(RunStatus::Gating);
11020    }
11021    let last = reviews.last()?;
11022    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11023    if reviews.len() < max_rounds && !stagnant {
11024        return None;
11025    }
11026    if last.incomplete() && last.blocking == 0 {
11027        return Some(RunStatus::Blocked);
11028    }
11029    if last.e2e_status() == E2eStatus::ResourceBlocked
11030        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11031    {
11032        return None;
11033    }
11034    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11035        RunStatus::Gating
11036    } else {
11037        RunStatus::Blocked
11038    })
11039}
11040
11041/// How long a re-ask may take, given the budget the first attempt had.
11042///
11043/// A `nudged` retry is a request to restate an answer the seat has already
11044/// worked out: it carries no new work, so it does not deserve the original
11045/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11046/// 133 seconds while a third sat for over ten minutes on a resumed session
11047/// holding 410 KB of prior output - and because the retry had inherited the
11048/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11049/// judging round whose other seats were long finished.
11050///
11051/// A quarter of the budget, with a floor so that a deliberately short timeout
11052/// does not collapse to nothing. A retry that re-sends the whole prompt
11053/// (because the seat kept no context) is the original job again, and keeps the
11054/// original budget.
11055pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11056    if nudged {
11057        (full / 4).max(Duration::from_secs(120)).min(full)
11058    } else {
11059        full
11060    }
11061}
11062
11063/// Run a wave and parse each reply, re-asking the seats whose reply was
11064/// unusable.
11065///
11066/// The re-ask is a nudge rather than the whole prompt again when the seat still
11067/// holds its conversation, which is the difference between a cheap retry and
11068/// paying for the entire candidate set twice.
11069///
11070/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11071/// successor in `roster` is handed to it instead of being re-asked: the
11072/// handover is the retry. A seat with no successor left (a single-agent
11073/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11074/// the asks to one seat in one node number at most
11075/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11076/// is asked once (a dropped stream is nudged first), and only the last agent
11077/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11078/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11079/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11080/// other failure stops the chain when the previous agent failed the same way
11081/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11082/// [`has_context`] is false and the job's own full prompt and full budget are
11083/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11084/// intermediate ones are not losses) and is returned as a failure like any
11085/// other absent seat — the caller decides whether the panel still has a
11086/// quorum. An empty `roster` disables handover: failures are nudged as they
11087/// always were, and a quota is simply lost. A reply that fails to parse or
11088/// validate is the prompt's doing and is only ever nudged, never handed over.
11089///
11090/// The returned [`SeatState`] names the agent that answered (or tried last).
11091#[allow(clippy::too_many_arguments)]
11092async fn ask_json_wave<T>(
11093    jobs: Vec<SeatJob>,
11094    sem: Arc<Semaphore>,
11095    retries: usize,
11096    roster: &[AgentSpec],
11097    ctx: &WaveCtx<'_>,
11098    losses: &mut Vec<QuotaLoss>,
11099    state: &mut RunState,
11100    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11101) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11102where
11103    T: serde::de::DeserializeOwned + Send + 'static,
11104{
11105    ask_wave_with(
11106        jobs,
11107        sem,
11108        retries,
11109        roster,
11110        ctx,
11111        losses,
11112        state,
11113        &|text: &str| {
11114            let v = verdict::extract_json::<T>(text)?;
11115            validate(&v)?;
11116            Ok(v)
11117        },
11118    )
11119    .await
11120}
11121
11122/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11123/// a node whose answer is prose (deliberation) shares the same handover,
11124/// failure classification, quota bookkeeping and bounds instead of a copy.
11125///
11126/// A seat handed to another roster agent is sent the job's `handover` prompt
11127/// (when it has one) rather than `prompt`: the new agent has no session, so a
11128/// resume-style prompt would be incomplete. That holds for the handover ask
11129/// and for every nudge to that agent whose `has_context` is false.
11130#[allow(clippy::too_many_arguments)]
11131async fn ask_wave_with<T>(
11132    jobs: Vec<SeatJob>,
11133    sem: Arc<Semaphore>,
11134    retries: usize,
11135    roster: &[AgentSpec],
11136    ctx: &WaveCtx<'_>,
11137    losses: &mut Vec<QuotaLoss>,
11138    state: &mut RunState,
11139    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11140) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11141where
11142    T: Send + 'static,
11143{
11144    let n = jobs.len();
11145    let originals: Vec<SeatJob> = jobs;
11146    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11147    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11148    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11149    // answer, N once it has gone through N nudges. Read back once this
11150    // returns, so a caller building a history record (`ReviewRecord`) can
11151    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11152    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11153    // field's own doc.
11154    let mut nudges: Vec<usize> = vec![0; n];
11155    // The agent now occupying each seat, the ids it has already been through,
11156    // where in the roster the walk began, the class of the last failure, and
11157    // the stem word of a handover not yet asked (full prompt, full budget).
11158    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11159    let mut tried: Vec<BTreeSet<String>> = specs
11160        .iter()
11161        .map(|s| BTreeSet::from([s.id.clone()]))
11162        .collect();
11163    let starts: Vec<usize> = specs
11164        .iter()
11165        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11166        .collect();
11167    let carry = ctx.carry_seats && !roster.is_empty();
11168    // Carried across rounds: ids that failed the seat earlier, and how the
11169    // last failure went (so a repeat of it is not handed over again).
11170    let carried: Vec<BTreeSet<String>> = originals
11171        .iter()
11172        .map(|j| {
11173            state
11174                .seat_history
11175                .get(&j.seat.key)
11176                .filter(|_| carry)
11177                .map(|h| h.failed.clone())
11178                .unwrap_or_default()
11179        })
11180        .collect();
11181    let mut prev: Vec<Option<FailClass>> = originals
11182        .iter()
11183        .map(|j| {
11184            state
11185                .seat_history
11186                .get(&j.seat.key)
11187                .filter(|_| carry)
11188                .and_then(|h| h.last_fail.clone())
11189        })
11190        .collect();
11191    let next_agent =
11192        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11193            let others: BTreeSet<String> = specs
11194                .iter()
11195                .enumerate()
11196                .filter(|(j, _)| *j != i)
11197                .map(|(_, s)| s.id.clone())
11198                .collect();
11199            pick_successor(
11200                roster,
11201                starts[i],
11202                tried,
11203                carry.then(|| &carried[i]),
11204                &others,
11205            )
11206            .cloned()
11207        };
11208    let mut fresh: Vec<Option<String>> = vec![None; n];
11209    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11210    let mut pending: Vec<usize> = (0..n).collect();
11211
11212    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11213    // asks: an agent with a successor is asked once and handed over, and only
11214    // a seat with no successor spends `retries` nudges on the same agent. This
11215    // only guarantees the loop's own termination whatever those say.
11216    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11217    for round in 0..max_rounds {
11218        if pending.is_empty() {
11219            break;
11220        }
11221        let mut batch = Vec::with_capacity(pending.len());
11222        let mut renudged: Vec<&str> = Vec::new();
11223        for &i in &pending {
11224            let src = &originals[i];
11225            // A seat now held by another agent than the job named has no
11226            // session of its own: it gets the full-context prompt whenever
11227            // it is asked in full (the handover ask, a nudge it cannot
11228            // resume).
11229            let full: &str = match &src.handover {
11230                Some(h) if specs[i].id != src.spec.id => h,
11231                _ => &src.prompt,
11232            };
11233            // The prompt and the budget are one decision: a nudge restates
11234            // finished work, a re-sent prompt redoes it.
11235            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11236                (
11237                    full.to_owned(),
11238                    src.timeout,
11239                    format!("{}-{word}-{}", src.stem, specs[i].id),
11240                )
11241            } else if nudges[i] == 0 {
11242                (full.to_owned(), src.timeout, src.stem.clone())
11243            } else {
11244                renudged.push(src.seat.key.as_str());
11245                let why = done[i]
11246                    .as_ref()
11247                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11248                    .unwrap_or_else(|| "no parsable answer".to_owned());
11249                let nudge = prompt::nudge(&why);
11250                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11251                let prompt = if nudged {
11252                    nudge
11253                } else {
11254                    format!("{full}\n\n---\n\n{nudge}")
11255                };
11256                (
11257                    prompt,
11258                    retry_budget(src.timeout, nudged),
11259                    format!("{}-retry{}", src.stem, nudges[i]),
11260                )
11261            };
11262            batch.push(SeatJob {
11263                spec: specs[i].clone(),
11264                seat: seats[i].clone(),
11265                cwd: src.cwd.clone(),
11266                prompt,
11267                timeout,
11268                allow_write: src.allow_write,
11269                sessions: src.sessions,
11270                artifacts: src.artifacts.clone(),
11271                stem,
11272                handover: None,
11273            });
11274        }
11275
11276        if !renudged.is_empty() {
11277            state.event(
11278                ctx.node,
11279                format!("retry {round}: re-asking {}", renudged.join(", ")),
11280            );
11281        }
11282        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11283        let mut still = Vec::new();
11284        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11285            seats[i] = seat;
11286            // No agent was asked: nothing to hand over, nudge, remember or
11287            // charge. The caller sees `resource_block` and stops the node.
11288            if let AgentOutcome::ResourceBlocked(why) = &out {
11289                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11290                continue;
11291            }
11292            let class = FailClass::of(&out);
11293            // A dropped stream is nudged first (the conversation is still
11294            // there to pick up); only a seat whose nudges are spent hands over.
11295            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11296                && nudges[i] < retries
11297                && !roster.is_empty();
11298            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11299                let next = should_hand_over(prev[i].as_ref(), &cur)
11300                    .then(|| next_agent(i, &tried[i], &specs))
11301                    .flatten();
11302                if carry {
11303                    let h = state
11304                        .seat_history
11305                        .entry(originals[i].seat.key.clone())
11306                        .or_default();
11307                    h.failed.insert(specs[i].id.clone());
11308                    h.last_fail = Some(cur.clone());
11309                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11310                        h.last_ok = None;
11311                    }
11312                    // Saved before the next agent is asked, so a restart in
11313                    // between does not forget who failed.
11314                    if let Err(e) = state.save() {
11315                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11316                    }
11317                }
11318                if let Some(next) = next {
11319                    record_handover(
11320                        state,
11321                        ctx.node,
11322                        &originals[i].seat.key,
11323                        &specs[i].id,
11324                        &next.id,
11325                        &cur,
11326                        &fail_reason(&out),
11327                    );
11328                    tried[i].insert(next.id.clone());
11329                    prev[i] = Some(cur.clone());
11330                    seats[i] =
11331                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11332                    specs[i] = next;
11333                    fresh[i] = Some(cur.stem_word().to_owned());
11334                    nudges[i] = 0;
11335                    done[i] = Some(Err(anyhow::anyhow!(
11336                        "handed over after: {}",
11337                        fail_reason(&out)
11338                    )));
11339                    still.push(i);
11340                    continue;
11341                }
11342            }
11343            if carry
11344                && !nudge_first
11345                && let Some(cur) = class.clone()
11346            {
11347                // The chain ended here (no successor, or a repeated failure).
11348                let h = state
11349                    .seat_history
11350                    .entry(originals[i].seat.key.clone())
11351                    .or_default();
11352                h.failed.insert(specs[i].id.clone());
11353                h.last_fail = Some(cur);
11354            }
11355            let parsed = match out {
11356                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11357                AgentOutcome::Quota(o) => {
11358                    last_quota[i] = o.quota.clone();
11359                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11360                }
11361                // Not a parseable answer: the nudge loop re-asks it, which is
11362                // exactly what a dropped stream needs. Just don't hand its raw
11363                // error JSON to `extract_json`.
11364                AgentOutcome::Dropped(o) => {
11365                    let why = o
11366                        .dropped
11367                        .as_ref()
11368                        .map(|d| d.why.as_str())
11369                        .unwrap_or("the CLI ended the stream without delivering its answer");
11370                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11371                }
11372                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11373                    Err(anyhow::anyhow!(e))
11374                }
11375            };
11376            let quota = class == Some(FailClass::Quota);
11377            let failed = parsed.is_err();
11378            if !failed && matches!(ctx.node, "judge" | "vote") {
11379                // A validated answer restores this seat's quorum presence,
11380                // including on the normal path after an all-seat block.
11381                state
11382                    .quota
11383                    .retain(|loss| loss.seat != originals[i].seat.key);
11384            }
11385            done[i] = Some(parsed);
11386            if carry && !failed {
11387                let h = state
11388                    .seat_history
11389                    .entry(originals[i].seat.key.clone())
11390                    .or_default();
11391                h.failed.remove(&specs[i].id);
11392                h.last_ok = Some(specs[i].id.clone());
11393                h.last_fail = None;
11394            }
11395            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11396            // fail the same way; and never re-ask a seat that already parsed.
11397            // A failed agent that still has a successor is not re-asked
11398            // either: the handover was its remedy and has just been refused
11399            // (the chain stops on a repeated failure class). A seat with no
11400            // successor left (a single-agent roster, the roster's tail, or an
11401            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11402            let agent_failure = class.is_some()
11403                && !nudge_first
11404                && !roster.is_empty()
11405                && next_agent(i, &tried[i], &specs).is_some();
11406            if failed && !quota && !agent_failure && nudges[i] < retries {
11407                nudges[i] += 1;
11408                still.push(i);
11409            }
11410        }
11411        pending = still;
11412    }
11413
11414    // One loss per seat whose chain ended on a quota: the intermediate ones
11415    // were absorbed by a handover and are not losses.
11416    for (i, q) in last_quota.into_iter().enumerate() {
11417        if let Some(quota) = q {
11418            losses.push(QuotaLoss::from_report(
11419                originals[i].seat.key.clone(),
11420                ctx.node,
11421                Some(&quota),
11422            ));
11423        }
11424    }
11425
11426    // A stalled quorum already has its own recovery checkpoint. Refresh
11427    // its missing seats before deciding whether it can resume.
11428    if state.status != RunStatus::Stalled
11429        && ctx.node != "deliberate"
11430        && n > 0
11431        && losses.len() == n
11432        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11433    {
11434        block_machine_outage(state, ctx.node, losses);
11435        state.quota.append(losses);
11436    }
11437
11438    seats
11439        .into_iter()
11440        .zip(done)
11441        .zip(nudges)
11442        .map(|((seat, res), attempts)| {
11443            (
11444                seat,
11445                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11446                attempts,
11447            )
11448        })
11449        .collect()
11450}
11451
11452/// A node with no available agent has no task verdict. Preserve the node
11453/// checkpoint and record why a subsequent explicit resume is needed.
11454pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11455    let mut auth: Vec<_> = losses
11456        .iter()
11457        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11458        .map(|loss| loss.agent.clone())
11459        .collect();
11460    auth.sort();
11461    auth.dedup();
11462    let (kind, detail) = if auth.is_empty() {
11463        (
11464            "agent usage limit",
11465            "All seats reached their agent usage limits; resume after usage is available"
11466                .to_owned(),
11467        )
11468    } else {
11469        (
11470            "agent authentication",
11471            format!(
11472                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11473                auth.join(", ")
11474            ),
11475        )
11476    };
11477    state.resource_block = Some(crate::run::ResourceBlock {
11478        node: node.to_owned(),
11479        kind: kind.to_owned(),
11480        cause: crate::run::ResourceBlockCause::Other,
11481        detail: detail.clone(),
11482        at: Timestamp::now(),
11483    });
11484    state.event(node, detail);
11485}
11486
11487/// Did the wave end without asking any agent (see [`block_wave`])?
11488fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11489    results
11490        .iter()
11491        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11492}
11493
11494/// Settle a wave that never spawned anything: every job answers
11495/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11496/// else about the seat (session, history, quota) is touched.
11497fn block_wave(
11498    jobs: Vec<SeatJob>,
11499    state: &mut RunState,
11500    node: &str,
11501    attempt: usize,
11502    why: &str,
11503) -> Vec<(usize, SeatState, AgentOutcome)> {
11504    state.resource_wait = None;
11505    state
11506        .active
11507        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11508    if let Err(e) = state.save() {
11509        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11510    }
11511    jobs.into_iter()
11512        .enumerate()
11513        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11514        .collect()
11515}
11516
11517/// Acquire the shared build cache's lease, waiting out contention within
11518/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11519/// unbounded wait is never acceptable).
11520///
11521/// A first, non-blocking check happens before ever waiting; if it finds the
11522/// lease busy, that fact is logged as a `verify` event *and* flushed with
11523/// [`RunState::save`] immediately — not only once the wait finally succeeds
11524/// or gives up — so a `magi show` run by a different process while this one
11525/// is still waiting reads a `run.json` that says so, rather than whatever it
11526/// looked like before the wait started. The same applies to the terminal
11527/// failure: logged and saved before this returns `Err`, so a caller that
11528/// could not get the lease at all still leaves a legible record of why.
11529async fn acquire_cache_lease(
11530    state: &mut RunState,
11531    cache_dir: &Path,
11532    owner: &crate::cache::Owner,
11533    budget: Duration,
11534    node: &str,
11535    context: &str,
11536    cancel: &(dyn Fn() -> bool + Sync),
11537) -> Result<Option<crate::cache::Guard>> {
11538    let home = crate::run::home();
11539    let started = Instant::now();
11540    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11541        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11542        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11543        Err(e) => {
11544            state.event(
11545                "verify",
11546                format!("{context}: could not check the shared build cache: {e:#}"),
11547            );
11548            if let Err(e2) = state.save() {
11549                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11550            }
11551            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11552        }
11553    };
11554    state.event(
11555        "verify",
11556        format!(
11557            "{context}: waiting for the shared build cache at {} ({})",
11558            cache_dir.display(),
11559            busy.describe()
11560        ),
11561    );
11562    // Seats of this node are waiting, not running: say so, so the countdown a
11563    // reader sees only ever measures an agent that has been spawned.
11564    state.resource_wait = Some(crate::run::ResourceWait {
11565        node: context.to_owned(),
11566        kind: "shared build cache".to_owned(),
11567        holder: busy.describe(),
11568        since: Timestamp::now(),
11569        limit_secs: budget.as_secs(),
11570    });
11571    if let Err(e) = state.save() {
11572        tracing::warn!("could not persist a cache wait: {e:#}");
11573    }
11574    let remaining = budget.saturating_sub(started.elapsed());
11575    let waited = crate::cache::wait_for_unless(
11576        &home,
11577        cache_dir,
11578        owner,
11579        remaining,
11580        Duration::from_secs(5),
11581        cancel,
11582    )
11583    .await;
11584    state.resource_wait = None;
11585    let cause = match &waited {
11586        Ok(Some(_)) => None,
11587        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11588        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11589            Some(crate::run::ResourceBlockCause::Timeout {
11590                waited_secs: started.elapsed().as_secs(),
11591                limit_secs: budget.as_secs(),
11592            })
11593        }
11594        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11595    };
11596    if let Some(cause) = cause {
11597        let detail = match &waited {
11598            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11599            Err(error) => format!("{error:#}"),
11600        };
11601        state.resource_block = Some(crate::run::ResourceBlock {
11602            cause,
11603            node: node.to_owned(),
11604            kind: "shared build cache".into(),
11605            detail,
11606            at: Timestamp::now(),
11607        });
11608        state.event(
11609            "resource",
11610            state.resource_block.as_ref().unwrap().description(),
11611        );
11612    }
11613    if let Err(error) = state.save() {
11614        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11615    }
11616    waited
11617}
11618
11619/// The lease state could not be read at all (unreadable lease file, a
11620/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11621/// known to be contended, so a wave degrades to running without a shared cache
11622/// instead of stopping on a hold that [`crate::triage`] could never release.
11623#[derive(Debug)]
11624struct CacheCheckError(String);
11625
11626impl std::fmt::Display for CacheCheckError {
11627    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11628        f.write_str(&self.0)
11629    }
11630}
11631
11632impl std::error::Error for CacheCheckError {}
11633
11634/// Keep a specific acquisition outcome when the caller adds command evidence.
11635fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11636    state
11637        .resource_block
11638        .get_or_insert_with(|| crate::run::ResourceBlock {
11639            cause: crate::run::ResourceBlockCause::Other,
11640            node: node.to_owned(),
11641            kind: "shared build cache".into(),
11642            detail: format!("{error:#}"),
11643            at: Timestamp::now(),
11644        });
11645    if let Err(error) = state.save() {
11646        tracing::warn!("could not persist cache resource block: {error:#}");
11647    }
11648}
11649
11650/// Run `body` — a verify command batch — while holding the shared build
11651/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11652/// never interleave with another borrower's build against the same
11653/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11654/// timeout, or a human's own `magi review`. See the `cache` module doc for
11655/// why this matters more than Cargo's own per-target locking covers — two
11656/// *different* worktrees building the same package name/version into one
11657/// cache directory is a staleness bug, not a lock contention one.
11658///
11659/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11660/// agent waves do. Repository scheduling also has its own bounded,
11661/// cancellable resource-wait policy. Contention must not consume the
11662/// verification budget: `body` receives the full `budget` after both leases
11663/// are acquired and the cache is fresh. Without a shared cache, only
11664/// repository coordination is acquired before handing over the full budget.
11665///
11666/// A lease that cannot be acquired within the resource wait is reported as a single
11667/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11668/// verification — the same shape a spawn failure already takes in
11669/// [`run_commands`], so a caller need not special-case it.
11670#[allow(clippy::too_many_arguments)]
11671async fn with_cache_lease<'s, F, Fut>(
11672    state: &'s mut RunState,
11673    cache_dir: Option<&Path>,
11674    node: &str,
11675    seat: &str,
11676    worktree: &Path,
11677    head: &str,
11678    budget: Duration,
11679    context: &str,
11680    wait: &crate::repo_lock::WaitPolicy,
11681    body: F,
11682) -> Result<(Vec<CommandOutcome>, bool)>
11683where
11684    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11685    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11686{
11687    // Match editing waves: cache first, repository second. In particular,
11688    // pre_gate must never hold the repository while waiting for a wave's cache.
11689    let started = Instant::now();
11690    let home = crate::run::home();
11691    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11692    let guard = if let Some(cache_dir) = cache_dir {
11693        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11694        match acquire_cache_lease(
11695            state,
11696            cache_dir,
11697            &owner,
11698            resource_limit,
11699            node,
11700            context,
11701            &|| wait.cancelled(),
11702        )
11703        .await
11704        .and_then(|g| {
11705            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11706        }) {
11707            Ok(g) => Some(g),
11708            Err(e) => {
11709                record_cache_error(state, node, &e);
11710                return Ok((
11711                    vec![CommandOutcome {
11712                        command: "(waiting for the shared build cache)".into(),
11713                        code: None,
11714                        output_tail: e.to_string(),
11715                        duration_ms: started.elapsed().as_millis() as u64,
11716                        resource_blocked: true,
11717                    }],
11718                    false,
11719                ));
11720            }
11721        }
11722    } else {
11723        None
11724    };
11725    let _coordination = coordinate_operation(state, worktree, wait).await?;
11726    if let Some(cache_dir) = cache_dir {
11727        let identity = crate::cache::Identity::new(worktree, head);
11728        if let Err(e) = crate::cache::ensure_fresh(
11729            &home,
11730            cache_dir,
11731            &identity,
11732            &state.config.verify.cargo_manifests(),
11733        ) {
11734            record_cache_error(state, node, &e);
11735            state.event(
11736                "verify",
11737                format!(
11738                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11739                    worktree.display(),
11740                    short(head)
11741                ),
11742            );
11743            if let Some(guard) = guard {
11744                guard.release();
11745            }
11746            return Ok((
11747                vec![CommandOutcome {
11748                    command: "(confirming the shared build cache is fresh)".into(),
11749                    code: None,
11750                    output_tail: e.to_string(),
11751                    duration_ms: started.elapsed().as_millis() as u64,
11752                    resource_blocked: true,
11753                }],
11754                false,
11755            ));
11756        }
11757    }
11758    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11759    if !timed_out_pids.is_empty() {
11760        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11761    }
11762    // Drop repository coordination before releasing the cache, reversing the
11763    // acquisition order. A cancelled/error future also drops both guards.
11764    drop(_coordination);
11765    if let Some(guard) = guard {
11766        guard.release();
11767    }
11768    Ok((outcomes, retried))
11769}
11770
11771/// Poll `pids` — commands [`run_commands`] reports as still running when its
11772/// own timeout elapsed — until every one is confirmed gone, or
11773/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11774///
11775/// Real confirmation where confirmation is possible, not a substitute for
11776/// full process-tree observation: a grandchild the timed-out process spawned
11777/// and that survives independently of it is invisible to a pid check the
11778/// same way it always was, and continuing to observe and collect *that*
11779/// stays a different piece of work with its own owner. This only narrows a
11780/// fixed blind wait into an actual check of the pids this process does know
11781/// about.
11782async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11783    wait_for_pids_with(
11784        pids,
11785        crate::proc::pid_alive,
11786        LEASE_RELEASE_POLL,
11787        LEASE_RELEASE_MAX_WAIT,
11788    )
11789    .await;
11790}
11791
11792/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11793/// interval and ceiling supplied by the caller, so the polling *logic* -
11794/// returns as soon as every pid reports dead, gives up at the ceiling
11795/// otherwise - is testable on millisecond durations without asking the real
11796/// OS about a pid at all.
11797async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11798    pids: &[u32],
11799    alive: F,
11800    poll: Duration,
11801    max_wait: Duration,
11802) {
11803    let deadline = Instant::now() + max_wait;
11804    loop {
11805        if pids.iter().all(|&pid| !alive(pid)) {
11806            return;
11807        }
11808        if Instant::now() >= deadline {
11809            return;
11810        }
11811        tokio::time::sleep(poll).await;
11812    }
11813}
11814
11815/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11816/// admission that it could not even get a verify command to run, as opposed
11817/// to evidence the command actually produced? A caller that would otherwise
11818/// read a resource-blocked outcome as a red command must check this first:
11819/// see [`Runner::gate`], which retries rather than records `Blocked` when
11820/// this is true.
11821fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11822    outcomes.iter().any(|o| o.resource_blocked)
11823}
11824
11825/// What [`Runner::gate_fix_round`] decided.
11826pub(crate) enum GateFix {
11827    /// The tree changed and `verify.e2e` is still green: run the gate again.
11828    Retry,
11829    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11830    /// last failure stands and the run ends blocked.
11831    Stop,
11832    /// `verify.e2e` could not run after the fix (magi's own contention):
11833    /// decide nothing now, a later reentry retries.
11834    Defer,
11835}
11836
11837/// Is every red command in `outcomes` an ordinary failure the code could
11838/// explain: it ran, exited non-zero, and said something?
11839///
11840/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11841/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11842/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11843/// what the command is stays the gate's business.
11844pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11845    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11846    red.peek().is_some()
11847        && red.all(|o| {
11848            !o.resource_blocked
11849                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11850                && !o.output_tail.trim().is_empty()
11851        })
11852}
11853
11854/// Describe one verify command's outcome for the event log, distinguishing a
11855/// build/link failure — the toolchain never produced a binary to run — from
11856/// an actual test failure, since only the latter is a verdict on the patch.
11857fn e2e_outcome_label(o: &CommandOutcome) -> String {
11858    if o.ok() {
11859        return "pass".to_owned();
11860    }
11861    let reason = if o.build_failed() {
11862        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11863    } else {
11864        format!("FAIL ({:?})", o.code)
11865    };
11866    format!(
11867        "{reason}\n{}",
11868        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11869    )
11870}
11871
11872/// Run `verify.e2e`, retrying once if the first attempt could not build or
11873/// link — a build/link failure is frequently a race against a shared
11874/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11875/// `verify` event per command, tagged with `context` (normally `"round N"`)
11876/// so the two call sites that need this — the ordinary per-round leg in
11877/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11878/// it will ever call a round green — read identically in the event log.
11879async fn run_e2e_with_retry(
11880    state: &mut RunState,
11881    shell: &[String],
11882    commands: &[String],
11883    worktree: &Path,
11884    timeout: Duration,
11885    context: &str,
11886) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11887    let (mut e2e, mut timed_out_pids) = run_commands(
11888        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11889    )
11890    .await;
11891    for o in &e2e {
11892        state.event(
11893            "verify",
11894            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11895        );
11896    }
11897    // A build/link failure is not a verdict on the patch — it is frequently a
11898    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11899    // one retry before letting a red like that decide the round.
11900    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11901    if verify_retried {
11902        state.event(
11903            "verify",
11904            format!(
11905                "{context}: verify could not build/link, not a test result — retrying once \
11906                 before concluding"
11907            ),
11908        );
11909        let retried = run_commands(
11910            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11911        )
11912        .await;
11913        e2e = retried.0;
11914        // Both attempts' timeouts matter, not just the last one: the first
11915        // attempt's descendants may still be alive alongside the retry's.
11916        timed_out_pids.extend(retried.1);
11917        for o in &e2e {
11918            state.event(
11919                "verify",
11920                format!(
11921                    "{context}: retry `{}` -> {}",
11922                    o.command,
11923                    e2e_outcome_label(o)
11924                ),
11925            );
11926        }
11927    }
11928    (e2e, verify_retried, timed_out_pids)
11929}
11930
11931/// Run configured shell commands in `cwd`, in order. The second element is
11932/// the pid of every command that hit `timeout` and was still running when
11933/// this stopped waiting on it (best-effort: `None` when the platform did not
11934/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
11935/// that releases a shared resource afterward needs to know.
11936///
11937/// Records `task` into [`RunState::active`] at every command boundary
11938/// (`RunState::task_command`) and clears it once the whole list has run
11939/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
11940/// for minutes with no seat and no output of its own to show for it (see
11941/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
11942/// apart from "not yet run" without this), and this is the only place that
11943/// knows which command is running right now and how many are left. Three
11944/// saves per command — start, not per second — matching the same "only at a
11945/// boundary" rule [`wave`] already follows for seats.
11946#[allow(clippy::too_many_arguments)]
11947async fn run_commands(
11948    state: &mut RunState,
11949    node: &str,
11950    task: &str,
11951    attempt: usize,
11952    shell: &[String],
11953    commands: &[String],
11954    cwd: &Path,
11955    timeout: Duration,
11956) -> (Vec<CommandOutcome>, Vec<u32>) {
11957    if commands.is_empty() {
11958        // Nothing to mark as running and nothing to clear — an empty list
11959        // means "not configured", and touching `active` (or the disk) over
11960        // that would be a write for every round of a repo with no
11961        // `verify.e2e` / `verify.gate` commands at all.
11962        return (Vec::new(), Vec::new());
11963    }
11964    let mut out = Vec::new();
11965    let mut timed_out_pids = Vec::new();
11966    let total = commands.len();
11967    for (idx, command) in commands.iter().enumerate() {
11968        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11969        if let Err(e) = state.save() {
11970            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11971        }
11972        let started = Instant::now();
11973        let mut cmd = tokio::process::Command::new(&shell[0]);
11974        cmd.quiet();
11975        // Verify the checkout as a standalone caller. Worker ownership and
11976        // pinned executable selection belong to this driver, not to tests or
11977        // tools it runs. Preserve home/seat context and Running's containment.
11978        for key in crate::worker::INVOCATION_ENV {
11979            cmd.env_remove(key);
11980        }
11981        cmd.args(&shell[1..])
11982            .arg(command)
11983            .current_dir(cwd)
11984            .stdin(std::process::Stdio::null())
11985            .stdout(std::process::Stdio::piped())
11986            .stderr(std::process::Stdio::piped())
11987            .kill_on_drop(true);
11988        let mut outputs = Vec::new();
11989        let protected = (|| -> Result<()> {
11990            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11991                outputs.push(guard);
11992            }
11993            if let Some(cache) = state.config.cache_dir() {
11994                outputs.push(crate::retention::OutputLease::cache(
11995                    &crate::run::home(),
11996                    &cache,
11997                    false,
11998                )?);
11999            }
12000            Ok(())
12001        })();
12002        if let Err(e) = protected {
12003            out.push(CommandOutcome {
12004                command: command.clone(),
12005                code: None,
12006                output_tail: format!("output ownership unavailable: {e:#}"),
12007                duration_ms: 0,
12008                resource_blocked: true,
12009            });
12010            continue;
12011        }
12012        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12013        let (code, body) = match spawned {
12014            Ok(mut running) => {
12015                let pid = running.child().id();
12016                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12017                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12018                let mut paths = vec![cwd.to_owned()];
12019                if let Some(cache) = state.config.cache_dir() {
12020                    paths.push(cache);
12021                }
12022                let mut pressure = None;
12023                let status = tokio::select! {
12024                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12025                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12026                        pressure = Some(reason); None
12027                    }
12028                };
12029                let (code, failure) = match status {
12030                    Some(Ok(Ok(status))) => (
12031                        status.code(),
12032                        status
12033                            .code()
12034                            .is_none()
12035                            .then(|| format!("command terminated: {status}")),
12036                    ),
12037                    Some(Ok(Err(error))) => {
12038                        (None, Some(format!("failed waiting for command: {error}")))
12039                    }
12040                    Some(Err(_)) => {
12041                        // Leased cleanup confirms process-tree death itself.
12042                        // Unleased commands still need the caller's PID check
12043                        // after kill-on-drop, before releasing cache ownership.
12044                        if !running.coordinated()
12045                            && let Some(pid) = pid
12046                        {
12047                            timed_out_pids.push(pid);
12048                        }
12049                        (
12050                            None,
12051                            Some(format!("timed out after {}s", timeout.as_secs())),
12052                        )
12053                    }
12054                    None => (None, None),
12055                };
12056                running.finish().await;
12057                if let Some(reason) = pressure {
12058                    state.resource_block = Some(crate::run::ResourceBlock {
12059                        cause: crate::run::ResourceBlockCause::Other,
12060                        node: node.to_owned(),
12061                        kind: "disk pressure".to_owned(),
12062                        detail: reason.clone(),
12063                        at: Timestamp::now(),
12064                    });
12065                    out.push(CommandOutcome {
12066                        command: command.clone(),
12067                        code: None,
12068                        output_tail: reason,
12069                        duration_ms: started.elapsed().as_millis() as u64,
12070                        resource_blocked: true,
12071                    });
12072                    break;
12073                }
12074                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12075                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12076                if let Some(failure) = failure {
12077                    body.push('\n');
12078                    body.push_str(&failure);
12079                }
12080                (code, body)
12081            }
12082            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12083        };
12084        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12085        if !body.is_empty() {
12086            // The full text is kept whatever the summary dropped. A fresh
12087            // name per command so a resume or retry never overwrites an
12088            // earlier run's output. A failed write changes no verdict.
12089            match crate::run::write_verification_artifact(state, task, &body) {
12090                Ok(name) if summary != body => {
12091                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12092                }
12093                Ok(_) => {}
12094                Err(e) => {
12095                    let warning = format!("could not save the full {task} output: {e:#}");
12096                    tracing::warn!("{warning}");
12097                    state.event(node, warning);
12098                    if summary != body {
12099                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12100                    }
12101                }
12102            }
12103        }
12104        out.push(CommandOutcome {
12105            command: command.clone(),
12106            code,
12107            output_tail: summary,
12108            duration_ms: started.elapsed().as_millis() as u64,
12109            resource_blocked: false,
12110        });
12111    }
12112    state.task_finished(task);
12113    if let Err(e) = state.save() {
12114        tracing::warn!("could not persist the end of {task}: {e:#}");
12115    }
12116    (out, timed_out_pids)
12117}
12118
12119/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12120/// section (`report::run`) and in the `merge` event this node records — for
12121/// the operator to run by hand.
12122///
12123/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12124/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12125/// merge commits", or "require linear history") rejects the push a `--no-ff`
12126/// merge would produce, which is exactly the guidance this function replaces.
12127/// `message`'s first line becomes the squash commit's subject, matching the
12128/// note `report::run` prints alongside this command — see that function for
12129/// why an explicit subject is not optional there.
12130fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12131    let repo = repo.display();
12132    match style {
12133        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12134        MergeStyle::Squash => {
12135            // The subject sits inside double quotes, and a title an agent
12136            // wrote may carry the characters that break out of them.
12137            let subject = message
12138                .lines()
12139                .next()
12140                .unwrap_or(branch)
12141                .replace(['\\', '"', '$', '`'], "");
12142            format!(
12143                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12144            )
12145        }
12146        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12147    }
12148}
12149
12150/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12151/// under the hood, rejects a `title` over 256 characters and the whole
12152/// command fails — no PR at all, for a run whose body was otherwise fine
12153/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12154/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12155/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12156/// this bound. It is a margin, not a guarantee — a title packed
12157/// with multi-unit characters could still in principle land close to the
12158/// edge, but a real task title's occasional emoji or accented letter fits
12159/// comfortably inside it.
12160const PR_TITLE_MAX: usize = 240;
12161
12162/// A pull request title from the opening line of `text`, or `None` when that
12163/// line is not English (GitHub text is) or has no letters.
12164///
12165/// A line within `max` is kept as is. A longer one is cut at the end of its
12166/// first sentence when that falls inside `max`, else at a word boundary with a
12167/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12168/// make every merely-truncated title look non-English). The language check
12169/// runs on the kept text before any mark is added, so only what GitHub will
12170/// show is judged: an English opening followed by non-ASCII far past the cut
12171/// still passes.
12172fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12173    let line = queue::first_line(text)?;
12174    let chars: Vec<char> = line.chars().collect();
12175    let (kept, mark) = if chars.len() <= max {
12176        (line.to_owned(), "")
12177    } else if let Some(end) = sentence_end(&chars, max) {
12178        (chars[..end].iter().collect::<String>(), "")
12179    } else {
12180        let room = max.saturating_sub(3);
12181        // Cut at the last space inside the room; when the char just past the
12182        // room is a space the room already ends on a word.
12183        let cut = if chars[room].is_whitespace() {
12184            room
12185        } else {
12186            chars[..room]
12187                .iter()
12188                .rposition(|c| c.is_whitespace())
12189                .unwrap_or(room)
12190        };
12191        let head: String = chars[..cut].iter().collect();
12192        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12193        (head.to_owned(), "...")
12194    };
12195    if kept.is_empty() {
12196        return None;
12197    }
12198    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12199        return None;
12200    }
12201    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12202        return None;
12203    }
12204    Some(format!("{kept}{mark}"))
12205}
12206
12207/// The char length of the first sentence of `chars` when it ends within `max`
12208/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12209/// abbreviations so `e.g. foo` does not end a title early.
12210fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12211    const MIN: usize = 20;
12212    for i in MIN..max.min(chars.len()) {
12213        if !matches!(chars[i], '.' | '!' | '?') {
12214            continue;
12215        }
12216        let Some(&next) = chars.get(i + 1) else {
12217            continue;
12218        };
12219        if !next.is_whitespace() {
12220            continue;
12221        }
12222        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12223        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12224            continue;
12225        }
12226        let word: String = chars[..i]
12227            .iter()
12228            .rev()
12229            .take_while(|c| !c.is_whitespace())
12230            .collect::<Vec<_>>()
12231            .into_iter()
12232            .rev()
12233            .collect();
12234        let word = word.to_ascii_lowercase();
12235        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12236            continue;
12237        }
12238        let end = chars[..i]
12239            .iter()
12240            .rposition(|c| !c.is_whitespace())
12241            .map_or(i, |p| p + 1);
12242        return Some(end);
12243    }
12244    None
12245}
12246
12247/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12248/// change: a title and a body describing what was *implemented*, not the task
12249/// that asked for it. A task reads as a request; a reader of the merged
12250/// history wants the change.
12251struct PrMessage {
12252    title: String,
12253    body: String,
12254}
12255
12256impl PrMessage {
12257    /// Title, blank line, body. The first line is the squash/merge commit
12258    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12259    /// has to stay one sensible line.
12260    fn commit_message(&self) -> String {
12261        format!("{}\n\n{}", self.title, self.body)
12262    }
12263}
12264
12265/// The text after a leading `TITLE:` (any case) on `line`.
12266fn title_marker(line: &str) -> Option<&str> {
12267    let line = line.trim();
12268    let head = line.get(..6)?;
12269    head.eq_ignore_ascii_case("title:")
12270        .then(|| line[6..].trim())
12271}
12272
12273/// The implementer's own one-line title: the `TITLE:` line the implement
12274/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12275/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12276/// source, and a title that says as much is refused here too.
12277fn summary_title(summary: &str) -> Option<String> {
12278    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12279    let raw = title_marker(first)?;
12280    if raw.is_empty() {
12281        return None;
12282    }
12283    let title = queue::title_from(raw, PR_TITLE_MAX);
12284    let lower = title.to_ascii_lowercase();
12285    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12286        return None;
12287    }
12288    Some(title)
12289}
12290
12291/// How `open_review`'s instruction begins; see [`landing_title`].
12292const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12293
12294/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12295pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12296
12297/// Does this run review work that already existed, rather than implement a
12298/// task? Runs recorded before `reviewed_commits` existed carry only the
12299/// candidate marker.
12300pub(crate) fn is_review_run(state: &RunState) -> bool {
12301    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12302}
12303
12304/// The title of a review-only run: the subject of the oldest commit under
12305/// review. Later commits are usually fixups, and `instruction` is the review
12306/// prompt, which says nothing about the change. GitHub text is English, so a
12307/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12308fn review_title(state: &RunState) -> Option<String> {
12309    english_subject(
12310        state.reviewed_commits.as_ref()?.first()?,
12311        state.config.graph.github_english(),
12312    )
12313}
12314
12315/// `raw` as a pull request title, or `None` when it is blank, not English
12316/// (GitHub text is), or one of magi's own candidate commit subjects.
12317fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12318    let raw = raw.trim();
12319    if raw.is_empty() {
12320        return None;
12321    }
12322    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12323        return None;
12324    }
12325    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12326        return None;
12327    }
12328    let title = queue::title_from(raw, PR_TITLE_MAX);
12329    let lower = title.to_ascii_lowercase();
12330    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12331        return None;
12332    }
12333    Some(title)
12334}
12335
12336/// What a review-only run's branch says about itself, read at the moment the
12337/// pull request is opened.
12338#[derive(Debug, Clone, PartialEq, Eq)]
12339struct BranchFacts {
12340    /// `(subject, body)` of each commit, oldest first.
12341    commits: Vec<(String, String)>,
12342    /// Trimmed `git diff --stat`.
12343    stat: String,
12344}
12345
12346/// Longest diff stat shown: this many file lines plus the summary line.
12347const STAT_FILE_LINES: usize = 25;
12348/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12349const COMMITS_MAX_CHARS: usize = 20_000;
12350
12351/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12352/// say or finds nothing, so the caller falls back to what the run recorded.
12353async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12354    let commits = git::commit_log(repo, base, branch).await.ok()?;
12355    if commits.is_empty() {
12356        return None;
12357    }
12358    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12359    let lines: Vec<&str> = stat.lines().collect();
12360    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12361        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12362        let more = format!(" ... {omitted} more file(s)");
12363        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12364        kept.push(&more);
12365        kept.push(lines[lines.len() - 1]);
12366        kept.join("\n")
12367    } else {
12368        lines.join("\n")
12369    };
12370    Some(BranchFacts { commits, stat })
12371}
12372
12373/// Defang what would break the surrounding markdown: a closing `</details>`
12374/// and a code fence.
12375fn markdown_safe(text: &str) -> String {
12376    text.replace("</details>", "&lt;/details&gt;")
12377        .replace("\x60\x60\x60", "~~~")
12378}
12379
12380fn neutral_title(state: &RunState, winner: char) -> String {
12381    format!(
12382        "chore: land candidate {} of run {}",
12383        winner.to_ascii_uppercase(),
12384        state.id
12385    )
12386}
12387
12388/// Reuse the graph's bounded fixer without starting another graph or driver.
12389/// Copy the working state back even on errors so spent rounds remain visible.
12390pub(crate) async fn repair_landing_gate(
12391    state: &mut RunState,
12392    winner: &Candidate,
12393    outcomes: &[CommandOutcome],
12394) -> Result<GateFix> {
12395    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12396        return Ok(GateFix::Stop);
12397    }
12398    let mut runner = Runner {
12399        roles: state.config.resolve_roles()?,
12400        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12401        state: state.clone(),
12402        pause: Pause::new(),
12403        interrupt: Pause::new(),
12404    };
12405    let result = runner.gate_fix_round(winner, outcomes).await;
12406    *state = runner.state;
12407    result
12408}
12409
12410/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12411/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12412pub(crate) async fn verify_current_head(
12413    state: &mut RunState,
12414    winner: &Candidate,
12415    wait: &crate::repo_lock::WaitPolicy,
12416) -> Result<bool> {
12417    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12418    // Preserve the previous record before replacing it, including on crash recovery.
12419    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12420        state
12421            .verification_history
12422            .push(crate::run::VerificationEvidence {
12423                superseded_by: Some(head.clone()),
12424                archived_at: Some(Timestamp::now()),
12425                fixes_spent: Some(state.gate_fixes.len()),
12426                gate_head: None,
12427                gate_ran: false,
12428                gate: Vec::new(),
12429                e2e: state.head_e2e.take(),
12430            });
12431    }
12432    if let Some(e) = &state.head_e2e
12433        && e.head == head
12434    {
12435        if e.passed {
12436            return Ok(true);
12437        }
12438        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12439            state.status = RunStatus::Blocked;
12440            state.save()?;
12441            return Ok(false);
12442        }
12443    } else if state.reviews.last().is_some_and(|r| {
12444        r.verified_head.as_deref() == Some(head.as_str())
12445            && (r.e2e_status() == E2eStatus::Passed
12446                || (r.e2e_status() == E2eStatus::NotConfigured
12447                    && state.config.verify.e2e.is_empty()))
12448    }) || state
12449        .gate_repair_e2e
12450        .as_ref()
12451        .is_some_and(|e| e.head == head && e.passed)
12452    {
12453        return Ok(true);
12454    }
12455    if let Some(e) = &state.gate_repair_e2e
12456        && e.head == head
12457        && !e.passed
12458        && !e.outcomes.is_empty()
12459        && !verify_inconclusive(&e.outcomes)
12460    {
12461        state.status = RunStatus::Blocked;
12462        state.save()?;
12463        return Ok(false);
12464    }
12465    state.head_e2e = Some(crate::run::GateRepairE2e {
12466        head: head.clone(),
12467        passed: false,
12468        outcomes: Vec::new(),
12469    });
12470    state.save()?;
12471    let commands = state.config.verify.e2e.clone();
12472    let shell = state.config.shell();
12473    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12474    let cache_dir = state.config.cache_dir();
12475    let outcomes = if commands.is_empty() {
12476        Vec::new()
12477    } else {
12478        let (outcomes, _) = with_cache_lease(
12479            state,
12480            cache_dir.as_deref(),
12481            "e2e",
12482            "e2e",
12483            &winner.worktree,
12484            &head,
12485            timeout,
12486            "current head e2e",
12487            wait,
12488            |state, budget| async move {
12489                run_e2e_with_retry(
12490                    state,
12491                    &shell,
12492                    &commands,
12493                    &winner.worktree,
12494                    budget,
12495                    "current head e2e",
12496                )
12497                .await
12498            },
12499        )
12500        .await?;
12501        outcomes
12502    };
12503    let unresolved = verify_inconclusive(&outcomes);
12504    let passed = !unresolved
12505        && outcomes.iter().all(CommandOutcome::ok)
12506        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12507    if !outcomes.is_empty() && !unresolved {
12508        state.record_base_progress(&head);
12509    }
12510    state.head_e2e = Some(crate::run::GateRepairE2e {
12511        head: head.clone(),
12512        passed,
12513        outcomes,
12514    });
12515    if !passed && !unresolved {
12516        state.status = RunStatus::Blocked;
12517        state.event(
12518            "e2e",
12519            format!("verify.e2e failed on head {head}; nothing published"),
12520        );
12521    }
12522    state.save()?;
12523    Ok(passed)
12524}
12525
12526/// Recheck a repair's e2e requirement across failures, resource deferrals and
12527/// process interruptions. A successful result only applies to its exact SHA.
12528pub(crate) async fn check_gate_repair_e2e(
12529    state: &mut RunState,
12530    winner: &Candidate,
12531    wait: &crate::repo_lock::WaitPolicy,
12532) -> Result<GateFix> {
12533    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12534        return Ok(GateFix::Retry);
12535    }
12536    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12537    if state
12538        .gate_repair_e2e
12539        .as_ref()
12540        .is_some_and(|r| r.head == head && r.passed)
12541    {
12542        return Ok(GateFix::Retry);
12543    }
12544    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12545        head: head.clone(),
12546        passed: false,
12547        outcomes: Vec::new(),
12548    });
12549    state.save()?;
12550    let commands = state.config.verify.e2e.clone();
12551    let outcomes = if commands.is_empty() {
12552        Vec::new()
12553    } else {
12554        let shell = state.config.shell();
12555        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12556        let cache_dir = state.config.cache_dir();
12557        let (outcomes, _) = with_cache_lease(
12558            state,
12559            cache_dir.as_deref(),
12560            "e2e",
12561            "e2e",
12562            &winner.worktree,
12563            &head,
12564            timeout,
12565            "gate repair e2e",
12566            wait,
12567            |state, budget| async move {
12568                run_e2e_with_retry(
12569                    state,
12570                    &shell,
12571                    &commands,
12572                    &winner.worktree,
12573                    budget,
12574                    "gate repair e2e",
12575                )
12576                .await
12577            },
12578        )
12579        .await?;
12580        outcomes
12581    };
12582    let deferred = verify_inconclusive(&outcomes);
12583    let passed = outcomes.iter().all(CommandOutcome::ok)
12584        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12585    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12586        head,
12587        passed,
12588        outcomes,
12589    });
12590    state.save()?;
12591    if deferred {
12592        Ok(GateFix::Defer)
12593    } else if !passed {
12594        state.event(
12595            "gate",
12596            "verify.e2e failed after gate repair; nothing published",
12597        );
12598        Ok(GateFix::Stop)
12599    } else {
12600        Ok(GateFix::Retry)
12601    }
12602}
12603
12604/// Run the same configured gate for a landing head and a fresh graph.
12605pub(crate) async fn run_landing_gate(
12606    state: &mut RunState,
12607    winner: &Candidate,
12608    wait: &crate::repo_lock::WaitPolicy,
12609) -> Result<Vec<CommandOutcome>> {
12610    let shell = state.config.shell();
12611    let gate_commands = state.config.verify.gate.clone();
12612    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12613    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12614        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12615        Vec::new()
12616    } else {
12617        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12618        let cache_dir = state.config.cache_dir();
12619        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12620        let (outcomes, _) = with_cache_lease(
12621            state,
12622            cache_dir.as_deref(),
12623            "gate",
12624            "gate",
12625            &winner.worktree,
12626            &head,
12627            timeout,
12628            "final gate",
12629            wait,
12630            |state, budget| {
12631                let shell = shell.clone();
12632                let gate_commands = gate_commands.clone();
12633                let winner = winner.clone();
12634                async move {
12635                    Runner::run_pre_gate(state, &winner).await;
12636                    // Bind command evidence after mechanical fixes, before commands start.
12637                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12638                    let (outcomes, timed_out_pids) = run_commands(
12639                        state,
12640                        "gate",
12641                        "gate",
12642                        0,
12643                        &shell,
12644                        &gate_commands,
12645                        &winner.worktree,
12646                        budget,
12647                    )
12648                    .await;
12649                    (outcomes, false, timed_out_pids)
12650                }
12651            },
12652        )
12653        .await?;
12654        outcomes
12655    };
12656    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12657    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12658        let mut outcomes = outcomes;
12659        outcomes.push(CommandOutcome {
12660            command: "gate head stability".into(), code: None,
12661            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12662            duration_ms: 0, resource_blocked: false,
12663        });
12664        return Ok(outcomes);
12665    }
12666    if outcomes.is_empty() {
12667        // Nothing configured to check — distinct from every other
12668        // silence in this run's event log, since an empty `gate` alone
12669        // no longer says whether the gate ran at all (see
12670        // `RunState::gate_ran`'s own doc).
12671        state.event("gate", "no gate commands configured; not tested");
12672    }
12673    for o in &outcomes {
12674        state.event(
12675            "gate",
12676            format!(
12677                "`{}` -> {}",
12678                o.command,
12679                if o.ok() {
12680                    "pass".to_owned()
12681                } else {
12682                    format!(
12683                        "FAIL ({:?})\n{}",
12684                        o.code,
12685                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12686                    )
12687                }
12688            ),
12689        );
12690    }
12691    Ok(outcomes)
12692}
12693
12694/// The pull request title to hand to `land::merge_subject`. A review-only run
12695/// opened by an earlier build titled its pull request with the review prompt;
12696/// that title is dropped (empty, so the fallback applies) rather than landed.
12697/// Any other title, including an operator's rename, passes through untouched,
12698/// and so does every title of a run that implements a task.
12699pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12700    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12701        ""
12702    } else {
12703        pr_title
12704    }
12705}
12706
12707/// What the squash subject falls back to when the pull request title is empty
12708/// or candidate-shaped: for a review-only run the derived title, never the
12709/// review prompt held in `instruction`.
12710pub fn landing_subject_source(state: &RunState) -> String {
12711    if is_review_run(state) {
12712        let winner = state.candidates.first().map_or('A', |c| c.label);
12713        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12714    }
12715    state.instruction.clone()
12716}
12717
12718/// `summary` without its `TITLE:` line, which the pull request title already
12719/// carries.
12720fn summary_without_title(summary: &str) -> String {
12721    let mut lines = summary.trim().lines().peekable();
12722    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12723        lines.next();
12724    }
12725    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12726}
12727
12728/// The pull request title and body for the winning candidate.
12729///
12730/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12731/// the task's own opening line via [`queue::title_from`] when there is none.
12732/// `state.instruction` can open with blank lines (`task_text` only rejects a
12733/// body that is blank *entirely*), which `title_from` skips.
12734///
12735/// Publishing uses an independent description generated from the final diff
12736/// and structured evidence. Internal summaries and reports remain local.
12737/// Non-publishing previews may display the original task and candidate summary.
12738/// The footer retains the run and candidate provenance.
12739#[cfg(test)]
12740fn pr_message(state: &RunState, winner: char) -> PrMessage {
12741    pr_message_with(state, winner, None)
12742}
12743
12744/// [`pr_message`] with what the branch of a review-only run says about itself.
12745/// `facts` is ignored for a run that implements a task.
12746#[cfg(test)]
12747fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12748    let raw = pr_message_raw(
12749        state,
12750        winner,
12751        facts,
12752        crate::pr_outcome::FinalHead::Unreported,
12753        ChangeNote::NotAsked,
12754    );
12755    let id = crate::scrub::Identity::current();
12756    PrMessage {
12757        title: crate::scrub::scrub(&raw.title, &id),
12758        body: crate::scrub::scrub(&raw.body, &id),
12759    }
12760}
12761
12762/// What the public description may say about the change itself.
12763#[derive(Debug, Clone, Copy)]
12764enum ChangeNote<'a> {
12765    /// A local preview or non-publishing merge mode: no public generation needed.
12766    NotAsked,
12767    /// A summary written from the final diff.
12768    Summary(&'a crate::run::ChangeSummary),
12769    /// A summary was asked for and could not be produced; the text says why.
12770    Unavailable(&'a str),
12771}
12772
12773fn pr_message_raw(
12774    state: &RunState,
12775    winner: char,
12776    facts: Option<&BranchFacts>,
12777    head: crate::pr_outcome::FinalHead<'_>,
12778    change: ChangeNote<'_>,
12779) -> PrMessage {
12780    if !matches!(change, ChangeNote::NotAsked) {
12781        let (title, mut body) = match change {
12782            ChangeNote::Summary(c) => (
12783                c.title.clone(),
12784                format!("## Summary\n\n{}\n\n", c.body.trim()),
12785            ),
12786            ChangeNote::Unavailable(why) => (
12787                neutral_title(state, winner),
12788                format!(
12789                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12790                ),
12791            ),
12792            ChangeNote::NotAsked => unreachable!(),
12793        };
12794        body.push_str(&public_pr_evidence(state, head));
12795        body.push_str(&format!(
12796            "\n---\nmagi:run/{} magi:candidate-{}\n",
12797            state.id,
12798            winner.to_ascii_lowercase()
12799        ));
12800        return PrMessage { title, body };
12801    }
12802    pr_preview(state, winner, facts, head)
12803}
12804
12805/// Append public facts from the run records after every prose generation,
12806/// including posting-gate rewrites. The agent cannot drop this provenance.
12807fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12808    let mut body = crate::pr_outcome::review_rounds(state);
12809    body.push_str(&crate::pr_outcome::final_verification(state, head));
12810    let open = state.last_round_findings();
12811    if !open.is_empty() {
12812        body.push_str("## Open review findings\n\n");
12813        for f in open {
12814            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12815        }
12816        body.push('\n');
12817    }
12818    if let Some(c) = state.change_summary.as_ref()
12819        && head == crate::pr_outcome::FinalHead::At(&c.head)
12820    {
12821        body.push_str(&format!(
12822            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12823            c.base, c.head
12824        ));
12825    }
12826    body
12827}
12828
12829/// Local-only preview of the operator's task and candidate summary.
12830fn pr_preview(
12831    state: &RunState,
12832    winner: char,
12833    facts: Option<&BranchFacts>,
12834    head: crate::pr_outcome::FinalHead<'_>,
12835) -> PrMessage {
12836    let summary = state
12837        .candidates
12838        .iter()
12839        .find(|c| c.label == winner)
12840        .map(|c| c.summary.as_str())
12841        .unwrap_or_default();
12842    // The fallback is the operator's own words and may not be English; GitHub
12843    // text always is, so a non-English task gets a neutral title instead.
12844    let review = is_review_run(state);
12845    let title = if review {
12846        facts
12847            .and_then(|f| {
12848                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12849            })
12850            .or_else(|| review_title(state))
12851            .or_else(|| {
12852                state
12853                    .candidates
12854                    .iter()
12855                    .find(|c| c.label == winner)
12856                    .filter(|c| !c.branch.starts_with("magi/"))
12857                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12858            })
12859            .unwrap_or_else(|| neutral_title(state, winner))
12860    } else {
12861        summary_title(summary).unwrap_or_else(|| {
12862            english_title(
12863                &state.instruction,
12864                PR_TITLE_MAX,
12865                state.config.graph.github_english(),
12866            )
12867            .unwrap_or_else(|| neutral_title(state, winner))
12868        })
12869    };
12870
12871    let mut body = String::new();
12872    let what = summary_without_title(summary);
12873    if !what.is_empty() {
12874        body.push_str("## Summary\n\n");
12875        body.push_str(&what);
12876        body.push_str("\n\n");
12877    }
12878
12879    // Review history uses structured facts across all rounds, never reports.
12880    body.push_str(&crate::pr_outcome::review_rounds(state));
12881    body.push_str(&crate::pr_outcome::final_verification(state, head));
12882    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12883
12884    let open = state.open_findings();
12885    if !open.is_empty() {
12886        body.push_str("## Open review findings\n\n");
12887        for f in &open {
12888            body.push_str(&format!(
12889                "- `{}` [{:?}] {}\n",
12890                f.id, f.severity, "Context retained in local review records."
12891            ));
12892        }
12893        body.push('\n');
12894    }
12895
12896    if let Some(fix) = fix
12897        && !fix.rejected.is_empty()
12898    {
12899        body.push_str("## Declined by the fixer\n\n");
12900        for r in &fix.rejected {
12901            body.push_str(&format!(
12902                "- `{}`: {}\n",
12903                r.id, "Reason retained in local review records."
12904            ));
12905        }
12906        body.push('\n');
12907    }
12908
12909    if review {
12910        // The review prompt is not the task; list what the branch carries.
12911        body.push_str("## Commits under review\n\n");
12912        if let Some(facts) = facts {
12913            let mut left = COMMITS_MAX_CHARS;
12914            for (i, (subject, text)) in facts.commits.iter().enumerate() {
12915                let mut entry = format!("- {}\n", markdown_safe(subject));
12916                for l in markdown_safe(text).lines() {
12917                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
12918                }
12919                if left == 0 {
12920                    body.push_str(&format!(
12921                        "- ... {} more commit(s)\n",
12922                        facts.commits.len() - i
12923                    ));
12924                    break;
12925                }
12926                if entry.len() > left {
12927                    // Even the first commit is cut: one huge body must not
12928                    // push the whole description past GitHub's limit.
12929                    let mut end = left;
12930                    while !entry.is_char_boundary(end) {
12931                        end -= 1;
12932                    }
12933                    entry.truncate(end);
12934                    entry.push_str("\n  ... (truncated)\n");
12935                    left = 0;
12936                } else {
12937                    left -= entry.len();
12938                }
12939                body.push_str(&entry);
12940            }
12941            if !facts.stat.trim().is_empty() {
12942                body.push_str(&format!(
12943                    "\n## Diff stat\n\n```\n{}\n```\n",
12944                    markdown_safe(facts.stat.trim())
12945                ));
12946            }
12947        } else {
12948            match &state.reviewed_commits {
12949                Some(subjects) => {
12950                    for s in subjects {
12951                        body.push_str(&format!("- {}\n", s.trim()));
12952                    }
12953                }
12954                None => {
12955                    // An older run kept only the prompt, with the commit list
12956                    // after its first paragraph.
12957                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12958                    body.push_str(rest.trim());
12959                    body.push('\n');
12960                }
12961            }
12962        }
12963    } else {
12964        let task = state.instruction.trim();
12965        let task = if task.is_empty() {
12966            "(empty task)"
12967        } else {
12968            task
12969        };
12970        body.push_str(&format!(
12971            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12972            task.replace("</details>", "&lt;/details&gt;")
12973        ));
12974    }
12975
12976    body.push_str(&format!(
12977        "\n---\nmagi:run/{} magi:candidate-{}\n",
12978        state.id,
12979        winner.to_ascii_lowercase()
12980    ));
12981
12982    PrMessage { title, body }
12983}
12984
12985/// Source imports have already been prepared from structured inputs. Legacy
12986/// inferred references must never instruct a seat to reconstruct those imports.
12987fn seeded_instruction(state: &RunState) -> String {
12988    let mut inputs = Vec::new();
12989    if let Some(start) = &state.sources.start_from {
12990        inputs.push(format!(
12991            "Start from `{}` at `{}` (including ancestry).",
12992            start.token, start.sha
12993        ));
12994    }
12995    for pick in &state.sources.pick_commits {
12996        inputs.push(format!(
12997            "Pick `{}` at `{}` (single commit).",
12998            pick.token, pick.sha
12999        ));
13000    }
13001    if inputs.is_empty() {
13002        state.instruction.clone()
13003    } else {
13004        format!(
13005            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13006            state.instruction,
13007            inputs.join("\n")
13008        )
13009    }
13010}
13011
13012/// Does the winner have no commits ahead of the base it would land on?
13013/// Any failure to find out reads as "not empty": the merge then behaves as it
13014/// always did rather than refusing on a guess.
13015async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13016    let base = &state.base_branch;
13017    let mut against = base.clone();
13018    if mode == MergeMode::Pr {
13019        // A pull request lands on the remote's base, never the local branch
13020        // of the same name: if that cannot be read, "not empty" is the safe
13021        // answer.
13022        let remote = &state.config.merge.remote;
13023        let tracking = format!("{remote}/{base}");
13024        let fetched = git::fetch(repo, remote, base).await;
13025        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13026            against = tracking;
13027        } else {
13028            return false;
13029        }
13030    }
13031    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13032}
13033
13034/// Why nothing was opened for an empty winner, with what the task's own
13035/// references resolved to.
13036fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13037    let mut detail = format!(
13038        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13039         nothing to open a pull request for"
13040    );
13041    detail.push_str(
13042        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13043    );
13044    detail
13045}
13046
13047/// What the `Pr` merge does once it knows whether the branch already has an
13048/// open pull request.
13049#[derive(Debug, PartialEq, Eq)]
13050enum PrPlan {
13051    Create,
13052    Adopt { url: String, title: String },
13053    Stop(String),
13054}
13055
13056/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13057/// a failed lookup -> stop with the real reason. Never guesses.
13058fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13059    match found {
13060        Ok(land::OpenPr::None) => PrPlan::Create,
13061        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13062        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13063            "several open pull requests exist for this branch, not picking one: {}",
13064            urls.join(" ")
13065        )),
13066        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13067    }
13068}
13069
13070/// `gh pr create`, returning the PR url.
13071pub(crate) async fn gh_pr_create(
13072    cwd: &Path,
13073    base: &str,
13074    head: &str,
13075    title: &str,
13076    body: &str,
13077) -> Result<String> {
13078    let out = tokio::process::Command::new("gh")
13079        .args([
13080            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13081        ])
13082        .current_dir(cwd)
13083        .quiet()
13084        .stdin(std::process::Stdio::null())
13085        .output()
13086        .await
13087        .context("spawn gh")?;
13088    if out.status.success() {
13089        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13090    } else {
13091        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13092    }
13093}
13094
13095/// Never force deletion of uncommitted work, including unknown legacy trees.
13096async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13097    match std::fs::symlink_metadata(path) {
13098        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13099            // A previous cleanup may have removed the tree before a branch
13100            // deletion or state save failed. Unregister it if still present.
13101            git::worktree_remove_clean(repo, path).await?;
13102            return Ok(());
13103        }
13104        Err(e) => return Err(e.into()),
13105        Ok(_) => {}
13106    }
13107    if !git::is_clean(path).await? {
13108        bail!(
13109            "worktree has uncommitted work; refusing removal: {}",
13110            path.display()
13111        );
13112    }
13113    if !git::worktree_remove_clean(repo, path).await? {
13114        bail!("worktree removal refused: {}", path.display());
13115    }
13116    Ok(())
13117}
13118
13119/// Tear a run's worktrees and branches down.
13120///
13121/// `home` is where the updated `run.json` is saved (via
13122/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13123/// a housekeeping pass already has its own honest `home` handed to it, and
13124/// falling through to the global here would write back through whichever
13125/// directory some other process or test pinned into that `OnceLock` first,
13126/// not the one the caller actually resolved its `runs` and `state` from.
13127pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13128    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13129        bail!("refusing fold: {reason}");
13130    }
13131    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13132        bail!("candidate preservation incomplete; refusing cleanup");
13133    }
13134    // Serialize ordinary folding with the same starts/resumes as target-only
13135    // eviction. Otherwise a second janitor could remove the whole worktree
13136    // around a live output borrower even though target eviction was fenced.
13137    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13138    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13139    let _driver = crate::driver::maintenance(home, &state.id)?;
13140    let mut output_guards = Vec::new();
13141    for entry in std::fs::read_dir(state.worktree_root())
13142        .into_iter()
13143        .flatten()
13144        .flatten()
13145    {
13146        if entry.path().join(".git").is_file() {
13147            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13148        }
13149    }
13150    let repo = state.repo.clone();
13151    // CLI, Web and automatic cleanup all enter here, independently of
13152    // Runner::fold_losers. Protect deletion and shared config release too.
13153    let wait = repository_wait(state, None);
13154    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13155        Ok(lease) => lease,
13156        Err(e) => {
13157            state.save_under(home)?;
13158            return Err(e);
13159        }
13160    };
13161    // Confirmed merges dispose of their owned sources after durable evidence
13162    // capture. Preflight every source before removing anything.
13163    let _source_outputs = if state.status == RunStatus::Merged
13164        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13165    {
13166        verify_merged_fold_sources(state, drop_winner, home).await?
13167    } else {
13168        Vec::new()
13169    };
13170    let root = state.worktree_root();
13171    let winner = state.tally.as_ref().map(|t| t.winner);
13172    let mut removed = Vec::new();
13173
13174    for i in 0..state.candidates.len() {
13175        let c = state.candidates[i].clone();
13176        let is_winner = Some(c.label) == winner;
13177        if is_winner && !drop_winner {
13178            continue;
13179        }
13180        let existed = c.worktree.exists();
13181        // A vanished directory can still have a Git registration that keeps
13182        // its branch checked out. Remove that registration before the branch.
13183        remove_preserved_worktree(&repo, &c.worktree).await?;
13184        if existed {
13185            removed.push(c.worktree.to_string_lossy().into_owned());
13186        }
13187        // A branch handed to a later run (and its pull request) is not this
13188        // run's to delete.
13189        let handed_over = state.released_branches.contains(&c.branch);
13190        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13191            if !git::branch_delete(&repo, &c.branch).await? {
13192                bail!("candidate branch removal refused");
13193            }
13194            removed.push(c.branch.clone());
13195        }
13196        state.candidates[i].folded = true;
13197    }
13198
13199    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13200        let path = name.path();
13201        let keep = !drop_winner
13202            && winner.is_some_and(|w| {
13203                path.file_name()
13204                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13205            });
13206        if keep {
13207            continue;
13208        }
13209        remove_preserved_worktree(&repo, &path).await?;
13210        removed.push(path.to_string_lossy().into_owned());
13211    }
13212
13213    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13214    // judge worktrees, so once the loop above has cleared all of them out,
13215    // the parent is a bare directory nobody else was ever going to remove -
13216    // git only ever managed what was inside it. Left alone, one of these
13217    // accumulates per fully-folded run; the operator's own machine had 74.
13218    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13219    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13220    // so does anything a run never claimed that happens to share the bay.
13221    remove_if_empty(&root);
13222
13223    if state.enabled_worktree_config && drop_winner {
13224        // A release, not a raw disable: some sibling run in this repository
13225        // may still hold its own reference (see `git::acquire_worktree_config`),
13226        // and only the last release actually turns the setting back off.
13227        git::release_worktree_config(&repo).await.ok();
13228        state.enabled_worktree_config = false;
13229    }
13230    state.save_under(home)?;
13231    Ok(removed)
13232}
13233
13234async fn verify_merged_fold_sources(
13235    state: &RunState,
13236    drop_winner: bool,
13237    home: &Path,
13238) -> Result<Vec<std::fs::File>> {
13239    let winner = state.tally.as_ref().map(|t| t.winner);
13240    let mut heads = BTreeSet::new();
13241    for candidate in &state.candidates {
13242        if Some(candidate.label) == winner && !drop_winner {
13243            continue;
13244        }
13245        anyhow::ensure!(
13246            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13247            "candidate recovery source is outside the inventoried run root: {}",
13248            candidate.worktree.display()
13249        );
13250        match std::fs::symlink_metadata(&candidate.worktree) {
13251            Ok(_) => {}
13252            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13253            Err(error) => return Err(error.into()),
13254        }
13255        if !state.released_branches.contains(&candidate.branch)
13256            && git::branch_exists(&state.repo, &candidate.branch).await?
13257        {
13258            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13259        }
13260    }
13261    let mut cargo_guards = Vec::new();
13262    let entries = match std::fs::read_dir(state.worktree_root()) {
13263        Ok(entries) => Some(entries),
13264        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13265        Err(error) => return Err(error.into()),
13266    };
13267    for entry in entries.into_iter().flatten() {
13268        let path = entry?.path();
13269        let metadata = std::fs::symlink_metadata(&path)?;
13270        anyhow::ensure!(
13271            metadata.is_dir() && !metadata.file_type().is_symlink(),
13272            "non-directory or symlink recovery source retained: {}",
13273            path.display()
13274        );
13275        if !drop_winner
13276            && winner.is_some_and(|w| {
13277                path.file_name()
13278                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13279            })
13280        {
13281            continue;
13282        }
13283        anyhow::ensure!(
13284            path.join(".git").is_file(),
13285            "unregistered recovery source retained: {}",
13286            path.display()
13287        );
13288        anyhow::ensure!(
13289            git::git(
13290                &path,
13291                &[
13292                    "status",
13293                    "--porcelain",
13294                    "--untracked-files=all",
13295                    "--ignored",
13296                    "--",
13297                    ".",
13298                    ":(exclude)target"
13299                ]
13300            )
13301            .await?
13302            .is_empty(),
13303            "source changes or ignored recovery files retained: {}",
13304            path.display()
13305        );
13306        match std::fs::symlink_metadata(path.join("target")) {
13307            Ok(_) => {
13308                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13309            }
13310            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13311            Err(error) => return Err(error.into()),
13312        }
13313        for checkpoint in [
13314            "rebase-merge",
13315            "rebase-apply",
13316            "sequencer",
13317            "CHERRY_PICK_HEAD",
13318            "MERGE_HEAD",
13319            "REBASE_HEAD",
13320            "REVERT_HEAD",
13321            "BISECT_LOG",
13322            "AUTO_MERGE",
13323            "MERGE_MSG",
13324        ] {
13325            let location = git::git(
13326                &path,
13327                &[
13328                    "rev-parse",
13329                    "--path-format=absolute",
13330                    "--git-path",
13331                    checkpoint,
13332                ],
13333            )
13334            .await?;
13335            anyhow::ensure!(
13336                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13337                if e.kind() == std::io::ErrorKind::NotFound),
13338                "Git recovery checkpoint retained: {checkpoint}"
13339            );
13340        }
13341        heads.insert(git::rev_parse(&path, "HEAD").await?);
13342    }
13343    crate::fold_evidence::preserve(state, home, &heads).await?;
13344
13345    Ok(cargo_guards)
13346}
13347
13348/// Remove `dir` if it exists and has nothing in it.
13349///
13350/// Best-effort and silent by design: a directory that is not empty (a run
13351/// whose winner is still parked there, a stray file some other process left)
13352/// is exactly the case this must refuse, and a directory that is already gone
13353/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13354/// already refuses a non-empty directory, so the emptiness check below is
13355/// belt, not suspenders - it is what keeps this from ever attempting the
13356/// removal in the case that matters, rather than trusting `remove_dir`'s
13357/// error path to have no side effects if it ever changed.
13358fn remove_if_empty(dir: &Path) {
13359    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13360        std::fs::remove_dir(dir).ok();
13361    }
13362}
13363
13364/// Severity of the worst open finding in the last review round, for reporting.
13365pub fn worst_open(state: &RunState) -> Option<Severity> {
13366    state
13367        .reviews
13368        .last()?
13369        .reviews
13370        .iter()
13371        .flat_map(|r| r.findings.iter())
13372        .map(|f| f.severity)
13373        .max()
13374}
13375
13376/// `[worktree] setup` hides its products per worktree, which needs
13377/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13378/// worktrees, like the hook's; every place that runs setup calls this first.
13379async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13380    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13381        git::acquire_worktree_config(repo).await?;
13382        state.enabled_worktree_config = true;
13383    }
13384    Ok(())
13385}
13386
13387#[cfg(test)]
13388mod tests {
13389    #[tokio::test]
13390    async fn launcher_base_fetch_requires_remote_success() {
13391        let tmp = tempfile::tempdir().unwrap();
13392        let repo = tmp.path().join("repo");
13393        let remote = tmp.path().join("remote");
13394        std::fs::create_dir(&repo).unwrap();
13395        std::fs::create_dir(&remote).unwrap();
13396        git::git(&remote, &["init", "--bare", "-b", "main"])
13397            .await
13398            .unwrap();
13399        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13400        git::git(
13401            &repo,
13402            &[
13403                "-c",
13404                "user.name=Fixture",
13405                "-c",
13406                "user.email=fixture@example.invalid",
13407                "commit",
13408                "--allow-empty",
13409                "-m",
13410                "fixture",
13411            ],
13412        )
13413        .await
13414        .unwrap();
13415        git::git(
13416            &repo,
13417            &["remote", "add", "origin", remote.to_str().unwrap()],
13418        )
13419        .await
13420        .unwrap();
13421        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13422        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13423        assert_eq!(
13424            resolve_base(&repo, "main", "origin", None)
13425                .await
13426                .unwrap()
13427                .commit,
13428            head
13429        );
13430        #[cfg(unix)]
13431        {
13432            let policy: crate::config::Coordination = toml::from_str(
13433                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13434            ).unwrap();
13435            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13436            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13437            let held = crate::repo_lock::SeatLease::operation_with(
13438                &repo,
13439                Some(&policy),
13440                &crate::repo_lock::WaitPolicy::default(),
13441            )
13442            .await
13443            .unwrap();
13444            assert!(held.is_some());
13445            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13446            tokio::pin!(fetch);
13447            assert!(
13448                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13449                    .await
13450                    .is_err()
13451            );
13452            assert!(
13453                !repo.join(".git/FETCH_HEAD").exists(),
13454                "no fetch before admission"
13455            );
13456            drop(held);
13457            let fetched = fetch.await.unwrap();
13458            assert_eq!(fetched.commit, head);
13459            assert_eq!(fetched.remote, "origin");
13460            assert_eq!(fetched.branch, "main");
13461        }
13462        // A surviving tracking ref is not successful fetch evidence.
13463        git::git(
13464            &repo,
13465            &[
13466                "remote",
13467                "set-url",
13468                "origin",
13469                tmp.path().join("missing").to_str().unwrap(),
13470            ],
13471        )
13472        .await
13473        .unwrap();
13474        let error = resolve_base(&repo, "main", "origin", None)
13475            .await
13476            .unwrap_err()
13477            .to_string();
13478        assert!(error.contains("cannot read origin/main"));
13479        assert!(error.contains("refusing to branch off"));
13480    }
13481
13482    #[test]
13483    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13484        let leaked = vec!["chore(deps): update a crate".to_owned()];
13485        let own = "fix(daemon): apply a chosen action";
13486        assert!(should_retitle("", own, &leaked));
13487        assert!(should_retitle(
13488            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13489            own,
13490            &leaked
13491        ));
13492        assert!(should_retitle(
13493            "chore: land candidate A of run 1",
13494            own,
13495            &leaked
13496        ));
13497        assert!(should_retitle(
13498            "magi: candidate A (uncommitted work)",
13499            own,
13500            &leaked
13501        ));
13502        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13503        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13504        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13505    }
13506
13507    #[test]
13508    fn pr_merge_plan_creates_adopts_or_stops() {
13509        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13510        assert_eq!(
13511            pr_merge_plan(Ok(land::OpenPr::One {
13512                url: "u".into(),
13513                title: "t".into()
13514            })),
13515            PrPlan::Adopt {
13516                url: "u".into(),
13517                title: "t".into()
13518            }
13519        );
13520        let PrPlan::Stop(many) =
13521            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13522        else {
13523            panic!("many must stop");
13524        };
13525        assert!(many.contains('a') && many.contains('b'));
13526        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13527            panic!("a failed lookup must stop");
13528        };
13529        assert!(err.contains("bad token"));
13530    }
13531
13532    use super::*;
13533    use crate::run::GateStatus;
13534    use std::collections::BTreeMap;
13535    use std::time::Duration;
13536
13537    fn conductor() -> AgentSpec {
13538        AgentSpec {
13539            id: "conductor".to_owned(),
13540            kind: crate::config::AgentKind::Command,
13541            model: None,
13542            command: vec!["true".to_owned()],
13543            extra_args: Vec::new(),
13544            env: BTreeMap::new(),
13545            prompt_delivery: None,
13546            max_turn_secs: None,
13547        }
13548    }
13549
13550    fn spec(id: &str) -> AgentSpec {
13551        AgentSpec {
13552            id: id.to_owned(),
13553            kind: crate::config::AgentKind::Command,
13554            model: None,
13555            command: vec!["true".to_owned()],
13556            extra_args: Vec::new(),
13557            env: BTreeMap::new(),
13558            prompt_delivery: None,
13559            max_turn_secs: None,
13560        }
13561    }
13562
13563    fn ids(xs: &[&str]) -> BTreeSet<String> {
13564        xs.iter().map(|s| (*s).to_owned()).collect()
13565    }
13566
13567    #[test]
13568    fn pick_successor_skips_an_agent_another_seat_holds() {
13569        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13570        let roster = [spec("a"), spec("b"), spec("c")];
13571        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13572        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13573    }
13574
13575    #[test]
13576    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13577        // The other seat started on c but was handed to d; c is free again.
13578        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13579        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13580        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13581        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13582        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13583    }
13584
13585    #[test]
13586    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13587        let roster = [spec("a"), spec("b")];
13588        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13589        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13590        let carried = ids(&["b"]);
13591        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13592        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13593    }
13594
13595    #[test]
13596    fn pick_successor_returns_none_without_any_untried_successor() {
13597        let roster = [spec("a"), spec("b")];
13598        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13599        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13600        let carried = ids(&["a"]);
13601        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13602    }
13603
13604    #[test]
13605    fn pick_successor_rescue_avoids_another_seats_occupant() {
13606        // b is a carried failure and free; a is held by the other seat.
13607        let roster = [spec("a"), spec("b"), spec("c")];
13608        let carried = ids(&["b", "c"]);
13609        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13610        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13611    }
13612
13613    #[test]
13614    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13615        let roster = [spec("a"), spec("b"), spec("c")];
13616        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13617        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13618    }
13619
13620    #[test]
13621    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13622        let roster = [spec("a"), spec("b"), spec("c")];
13623        let failed = ids(&["a", "b", "c"]);
13624        // Rescue looks at the whole roster, once per id, then runs out.
13625        let mut tried = ids(&["b"]);
13626        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13627        assert_eq!(first.id, "a");
13628        tried.insert(first.id.clone());
13629        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13630        assert_eq!(second.id, "c");
13631        tried.insert(second.id.clone());
13632        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13633    }
13634
13635    #[test]
13636    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13637        let roster = [spec("a"), spec("b")];
13638        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13639        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13640    }
13641
13642    #[test]
13643    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13644        let roster = [spec("a"), spec("b"), spec("c")];
13645        let h = SeatHistory {
13646            failed: ids(&["a"]),
13647            last_ok: Some("b".to_owned()),
13648            last_fail: None,
13649        };
13650        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13651        // A last answerer that left the roster, or later failed, is ignored.
13652        let gone = SeatHistory {
13653            last_ok: Some("zzz".to_owned()),
13654            ..h.clone()
13655        };
13656        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13657        let failed = SeatHistory {
13658            failed: ids(&["a", "b"]),
13659            last_ok: Some("b".to_owned()),
13660            last_fail: None,
13661        };
13662        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13663    }
13664
13665    #[test]
13666    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13667        let first = SeatState::new("review-1", "alpha", 7);
13668        let next = handover_seat("review-1", "gamma", 7);
13669        assert_ne!(first.claude_session, next.claude_session);
13670    }
13671
13672    #[test]
13673    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13674        let mut state = state_with_summary("x", "y");
13675        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13676        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13677        assert_ne!(a.claude_session, b.claude_session);
13678    }
13679
13680    #[test]
13681    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13682        let roster = [spec("a"), spec("b")];
13683        let h = SeatHistory {
13684            failed: ids(&["a", "b"]),
13685            ..SeatHistory::default()
13686        };
13687        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13688        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13689        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13690    }
13691
13692    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13693    // fallback loop depends on to terminate: it must walk forward from the
13694    // seat's own position, never restart at the front of the roster, and it
13695    // must never hand back an id already tried, however many times that id
13696    // happens to appear.
13697
13698    #[test]
13699    fn failure_signature_ignores_numbers_and_paths() {
13700        assert_eq!(
13701            failure_signature("exited with Some(2) and no usable output"),
13702            failure_signature("exited with Some(137) and no usable output")
13703        );
13704        assert_eq!(
13705            failure_signature("process exited with no usable output (exit code 2)"),
13706            failure_signature("process exited with no usable output (exit code 137)")
13707        );
13708        for (old, new) in [
13709            (
13710                "exited with Some(2) and no usable output",
13711                "process exited with no usable output (exit code 2)",
13712            ),
13713            (
13714                "exited with None and no usable output",
13715                "process exited with no usable output (exit code unavailable)",
13716            ),
13717        ] {
13718            let previous = FailClass::Other(failure_signature(old));
13719            let current = FailClass::Other(failure_signature(new));
13720            assert_eq!(previous, current, "resumption retains the failure class");
13721            assert!(!should_hand_over(Some(&previous), &current));
13722        }
13723        assert_eq!(
13724            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13725            failure_signature("cannot open /var/x.txt: denied")
13726        );
13727        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13728    }
13729
13730    #[test]
13731    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13732        let other = FailClass::Other("x".into());
13733        assert!(should_hand_over(None, &FailClass::Quota));
13734        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13735        assert!(should_hand_over(
13736            Some(&FailClass::Timeout),
13737            &FailClass::Timeout
13738        ));
13739        assert!(should_hand_over(None, &other));
13740        assert!(!should_hand_over(Some(&other), &other));
13741        assert!(should_hand_over(
13742            Some(&other),
13743            &FailClass::Other("y".into())
13744        ));
13745        // A quota or timeout in between ends the run of identical failures.
13746        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13747        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13748    }
13749
13750    #[test]
13751    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13752        let a = SeatState::new("judge-1", "alpha", 7);
13753        let b = handover_seat("judge-1", "beta", 7);
13754        assert_ne!(a.claude_session, b.claude_session);
13755        assert_eq!(b.turns, 0);
13756    }
13757
13758    #[test]
13759    fn a_timeout_is_classified_apart_from_other_failures() {
13760        assert_eq!(
13761            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13762            Some(FailClass::Timeout)
13763        );
13764        assert!(matches!(
13765            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13766            Some(FailClass::Other(_))
13767        ));
13768    }
13769
13770    #[test]
13771    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13772        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13773        let tried = BTreeSet::from(["beta".to_owned()]);
13774        // beta sits at index 1; the next candidate is gamma, never alpha —
13775        // which is very likely a different candidate slot's own agent.
13776        let next = next_untried_in_roster(&roster, 1, &tried);
13777        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13778    }
13779
13780    #[test]
13781    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13782        let roster = vec![spec("alpha"), spec("beta")];
13783        let tried = BTreeSet::from(["beta".to_owned()]);
13784        // beta is the roster's last entry: nothing follows it, and alpha —
13785        // earlier in the roster, almost certainly a different candidate
13786        // slot's own agent — must not be reached by wrapping back to it.
13787        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13788    }
13789
13790    #[test]
13791    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13792        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13793        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13794        // beta (index 1) and gamma (index 2, the only entry after it) have
13795        // both been tried; alpha (index 0) never has, but it comes before
13796        // beta's own position, so there is nothing further for this seat.
13797        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13798    }
13799
13800    #[test]
13801    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13802        let roster = vec![spec("a"), spec("a"), spec("b")];
13803        let tried = BTreeSet::from(["a".to_owned()]);
13804        let next = next_untried_in_roster(&roster, 0, &tried);
13805        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13806    }
13807
13808    #[test]
13809    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13810        let roster = vec![spec("a"), spec("b")];
13811        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13812        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13813    }
13814
13815    #[test]
13816    fn remove_if_empty_only_ever_takes_a_bare_directory() {
13817        let dir = tempfile::tempdir().unwrap();
13818        let bay = dir.path().join("ffff");
13819
13820        // Not there yet: nothing to do, nothing to panic on.
13821        remove_if_empty(&bay);
13822        assert!(!bay.exists());
13823
13824        // Something still inside - the winner's worktree, or a stray file -
13825        // keeps the directory standing.
13826        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13827        remove_if_empty(&bay);
13828        assert!(bay.exists(), "non-empty directory must survive");
13829
13830        // Once the last entry is gone, so is the directory itself.
13831        std::fs::remove_dir(bay.join("cand-A")).unwrap();
13832        remove_if_empty(&bay);
13833        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13834    }
13835
13836    // `round_is_clean` is the exact decision this task fixed: a round with a
13837    // seat that never answered must not read the same as a round every seat
13838    // actually reviewed. These are deterministic and process-free by design —
13839    // the equivalent end-to-end check (a real reviewer timing out under a
13840    // live graph run) is a genuine race against wall-clock contention, and a
13841    // spawn slow enough to blow even a generous budget under a loaded test
13842    // run must not turn this specific regression check flaky.
13843
13844    #[test]
13845    fn a_full_panel_that_found_nothing_is_clean() {
13846        assert!(round_is_clean(
13847            0,
13848            true,
13849            2,
13850            2,
13851            0,
13852            IncompleteReviewPolicy::Block
13853        ));
13854    }
13855
13856    #[test]
13857    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13858        assert!(!round_is_clean(
13859            0,
13860            true,
13861            1,
13862            2,
13863            0,
13864            IncompleteReviewPolicy::Block
13865        ));
13866    }
13867
13868    #[test]
13869    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13870        assert!(!round_is_clean(
13871            1,
13872            true,
13873            1,
13874            2,
13875            0,
13876            IncompleteReviewPolicy::Warn
13877        ));
13878    }
13879
13880    #[test]
13881    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13882        assert!(round_is_clean(
13883            0,
13884            true,
13885            1,
13886            2,
13887            0,
13888            IncompleteReviewPolicy::Warn
13889        ));
13890    }
13891
13892    #[test]
13893    fn a_full_panel_with_an_open_finding_is_not_clean() {
13894        assert!(!round_is_clean(
13895            1,
13896            true,
13897            2,
13898            2,
13899            0,
13900            IncompleteReviewPolicy::Block
13901        ));
13902    }
13903
13904    #[test]
13905    fn a_full_panel_with_a_red_e2e_is_not_clean() {
13906        assert!(!round_is_clean(
13907            0,
13908            false,
13909            2,
13910            2,
13911            0,
13912            IncompleteReviewPolicy::Block
13913        ));
13914    }
13915
13916    // The stall this task closes: under the default `block` policy, a seat
13917    // missing only because it was rate limited must not force a wait for a
13918    // session limit that will not lift by the next round. `round_is_clean`
13919    // is where that quorum carve-out lives; the review loop around it never
13920    // changes what a reviewer's vote or a finding's severity means.
13921
13922    #[test]
13923    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13924        // 1 of 2 answered, and the one missing was quota'd — the exact
13925        // "review-2 rate limited (quota)" shape from the field report.
13926        assert!(round_is_clean(
13927            0,
13928            true,
13929            1,
13930            2,
13931            1,
13932            IncompleteReviewPolicy::Block
13933        ));
13934    }
13935
13936    #[test]
13937    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13938        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
13939        // not a quota loss (`quota_missing` stays 0) — worth another try.
13940        assert!(!round_is_clean(
13941            0,
13942            true,
13943            1,
13944            2,
13945            0,
13946            IncompleteReviewPolicy::Block
13947        ));
13948    }
13949
13950    #[test]
13951    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13952        assert!(!round_is_clean(
13953            1,
13954            true,
13955            1,
13956            2,
13957            1,
13958            IncompleteReviewPolicy::Block
13959        ));
13960        assert!(!round_is_clean(
13961            0,
13962            false,
13963            1,
13964            2,
13965            1,
13966            IncompleteReviewPolicy::Block
13967        ));
13968    }
13969
13970    #[test]
13971    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13972        // Every seat quota'd, nobody answered: there is no panel to decide
13973        // on, so this must fall through to the existing block-and-retry
13974        // fallback rather than call an unreviewed patch clean.
13975        assert!(!round_is_clean(
13976            0,
13977            true,
13978            0,
13979            2,
13980            2,
13981            IncompleteReviewPolicy::Block
13982        ));
13983    }
13984
13985    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13986        CommandOutcome {
13987            command: "test".to_owned(),
13988            code,
13989            output_tail: String::new(),
13990            duration_ms: 0,
13991            resource_blocked,
13992        }
13993    }
13994
13995    #[test]
13996    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13997        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13998        assert!(
13999            !verify_inconclusive(&[outcome(Some(1), false)]),
14000            "an ordinary failure is still evidence about the patch"
14001        );
14002        assert!(verify_inconclusive(&[outcome(None, true)]));
14003        assert!(
14004            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14005            "one inconclusive outcome taints the whole batch"
14006        );
14007        assert!(!verify_inconclusive(&[]));
14008    }
14009
14010    #[tokio::test]
14011    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14012        // Alive for the first two checks, then dead - confirms the loop
14013        // actually re-polls rather than deciding once and sleeping out the
14014        // ceiling regardless.
14015        let calls = std::sync::atomic::AtomicUsize::new(0);
14016        let started = Instant::now();
14017        wait_for_pids_with(
14018            &[123],
14019            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14020            Duration::from_millis(5),
14021            Duration::from_secs(5),
14022        )
14023        .await;
14024        assert!(
14025            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14026            "must keep checking rather than deciding on the first answer"
14027        );
14028        assert!(
14029            started.elapsed() < Duration::from_secs(1),
14030            "must return the moment it is confirmed dead, not wait out the ceiling"
14031        );
14032    }
14033
14034    #[tokio::test]
14035    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14036        let started = Instant::now();
14037        wait_for_pids_with(
14038            &[123],
14039            |_| true, // never reports dead
14040            Duration::from_millis(5),
14041            Duration::from_millis(30),
14042        )
14043        .await;
14044        let elapsed = started.elapsed();
14045        assert!(
14046            elapsed >= Duration::from_millis(30),
14047            "must not give up before its own ceiling: {elapsed:?}"
14048        );
14049        assert!(
14050            elapsed < Duration::from_secs(1),
14051            "must not wait past its own ceiling either: {elapsed:?}"
14052        );
14053    }
14054
14055    #[tokio::test]
14056    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14057        let started = Instant::now();
14058        wait_for_pids_with(
14059            &[],
14060            |_| true,
14061            Duration::from_secs(5),
14062            Duration::from_secs(5),
14063        )
14064        .await;
14065        assert!(
14066            started.elapsed() < Duration::from_millis(200),
14067            "an empty pid list has nothing to confirm"
14068        );
14069    }
14070
14071    // `review_conclusion` is the exact decision the review hand-off task
14072    // fixed: a round budget spent (or a tree that stopped moving) must not
14073    // collapse into `Blocked` regardless of what verification actually
14074    // said. Deterministic and process-free for the same reason the
14075    // `round_is_clean` family above is.
14076    fn review_round(
14077        clean: bool,
14078        blocking: usize,
14079        answered: usize,
14080        expected: usize,
14081        progressed: bool,
14082        e2e_ok: bool,
14083    ) -> ReviewRound {
14084        ReviewRound {
14085            round: 1,
14086            head: "h".to_owned(),
14087            verified_head: None,
14088            verified_at: None,
14089            reviews: Vec::new(),
14090            e2e: vec![CommandOutcome {
14091                command: "test".to_owned(),
14092                code: Some(if e2e_ok { 0 } else { 1 }),
14093                output_tail: String::new(),
14094                duration_ms: 0,
14095                resource_blocked: false,
14096            }],
14097            verify_retried: false,
14098            e2e_deferred: false,
14099            e2e_defer_reason: None,
14100            fix: None,
14101            blocking,
14102            answered,
14103            expected,
14104            clean,
14105            progressed,
14106            vote_split: false,
14107            reconsideration: Vec::new(),
14108            verdict: None,
14109        }
14110    }
14111
14112    #[test]
14113    fn review_conclusion_is_none_when_nothing_has_run() {
14114        assert_eq!(review_conclusion(&[], 3), None);
14115    }
14116
14117    #[test]
14118    fn review_conclusion_is_none_while_rounds_remain() {
14119        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14120        assert_eq!(review_conclusion(&rounds, 3), None);
14121    }
14122
14123    #[test]
14124    fn review_conclusion_is_gating_once_a_round_is_clean() {
14125        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14126        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14127    }
14128
14129    #[test]
14130    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14131        let rounds = vec![
14132            review_round(false, 1, 2, 2, true, true),
14133            review_round(false, 1, 2, 2, true, true),
14134        ];
14135        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14136    }
14137
14138    #[test]
14139    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14140        let rounds = vec![
14141            review_round(false, 1, 2, 2, true, true),
14142            review_round(false, 1, 2, 2, true, false),
14143        ];
14144        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14145    }
14146
14147    #[test]
14148    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14149        // Magi never got a command to run against this round's own head — a
14150        // resource-blocked attempt, not a red one — so this must never
14151        // settle on `Blocked` the way a genuine e2e failure would. `None`
14152        // here is what tells `Runner::review_loop` to retry the check
14153        // itself rather than trust this cheap recomputation with a verdict
14154        // it cannot actually produce.
14155        let mut blocked = review_round(false, 1, 2, 2, true, false);
14156        blocked.e2e[0].resource_blocked = true;
14157        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14158        assert_eq!(review_conclusion(&rounds, 2), None);
14159    }
14160
14161    #[test]
14162    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14163        // Missing input, not a verified tree — never a hand-off candidate.
14164        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14165        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14166    }
14167
14168    #[test]
14169    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14170        let rounds = vec![
14171            review_round(false, 1, 2, 2, false, true),
14172            review_round(false, 1, 2, 2, false, true),
14173        ];
14174        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14175    }
14176
14177    fn secs(n: u64) -> Duration {
14178        Duration::from_secs(n)
14179    }
14180
14181    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14182    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14183    #[tokio::test]
14184    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14185        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14186        if std::env::var_os(CHILD).is_none() {
14187            // Other tests reconcile every run in the process-global home,
14188            // briefly taking owner locks. This ancestry test expects an idle
14189            // predecessor, so give it its own process and pinned test home.
14190            let output = std::process::Command::new(std::env::current_exe().unwrap())
14191                .args([
14192                    "--exact",
14193                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14194                    "--nocapture",
14195                ])
14196                .env(CHILD, "1")
14197                .env_remove("MAGI_WORKER_HOME")
14198                .env_remove("MAGI_WORKER_ID")
14199                .output()
14200                .unwrap();
14201            assert!(
14202                output.status.success()
14203                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14204                "{}\n{}",
14205                String::from_utf8_lossy(&output.stdout),
14206                String::from_utf8_lossy(&output.stderr)
14207            );
14208            return;
14209        }
14210        ask_test_home();
14211        let tmp = tempfile::tempdir().unwrap();
14212        let repo = tmp.path().join("repo");
14213        std::fs::create_dir_all(&repo).unwrap();
14214        init_repo(&repo);
14215        let g = |args: &[&str]| {
14216            let out = std::process::Command::new("git")
14217                .args(args)
14218                .current_dir(&repo)
14219                .quiet()
14220                .output()
14221                .unwrap();
14222            assert!(
14223                out.status.success(),
14224                "{}",
14225                String::from_utf8_lossy(&out.stderr)
14226            );
14227        };
14228        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14229        g(&["checkout", "-b", "historical"]);
14230        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14231        g(&["add", "."]);
14232        g(&["commit", "-m", "old alternative"]);
14233        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14234        g(&["checkout", "-b", "magi/task-selected", &base]);
14235        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14236        g(&["add", "."]);
14237        g(&["commit", "-m", "measurement infrastructure"]);
14238        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14239        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14240        g(&["add", "."]);
14241        g(&["commit", "-m", "dependent implementation"]);
14242        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14243        g(&["checkout", "main"]);
14244        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14245        g(&["add", "."]);
14246        g(&["commit", "-m", "unrelated base change"]);
14247        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14248        let origin = tmp.path().join("origin.git");
14249        g(&[
14250            "clone",
14251            "--bare",
14252            repo.to_str().unwrap(),
14253            origin.to_str().unwrap(),
14254        ]);
14255        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14256        g(&["fetch", "origin"]);
14257        let selection = crate::pr_recovery::Selection {
14258            url: "https://example.test/project/pull/1".into(),
14259            branch: "magi/task-selected".into(),
14260            head: head.clone(),
14261            source_head: None,
14262            source_run: None,
14263            source_proof: None,
14264            disposition: Some(crate::pr_recovery::Disposition::Review),
14265            instruction: "preserve measured behavior".into(),
14266            approved_at: jiff::Timestamp::now(),
14267        };
14268        let mut template = runner_at(RunStatus::Reviewing);
14269        template.state.repo = repo.clone();
14270        template.state.base_commit = current.clone();
14271        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14272        template.state.origin = Some(crate::run::Origin::queue("selected"));
14273        template.state.pr_head = Some(selection.branch.clone());
14274        template.state.pr_recovery = Some(selection.clone());
14275        let mut runner =
14276            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14277                .await
14278                .unwrap();
14279        let wt = runner.state.winner().unwrap().worktree.clone();
14280        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14281        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14282        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14283        assert!(runner.state.seeds.is_empty());
14284        assert_eq!(runner.state.instruction, "preserve measured behavior");
14285        runner.sync_to_base().await.unwrap();
14286        assert_eq!(
14287            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14288            "prerequisite"
14289        );
14290        assert_eq!(
14291            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14292            "measured implementation"
14293        );
14294        assert_eq!(
14295            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14296            "current main change"
14297        );
14298        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14299        assert_eq!(
14300            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14301            head
14302        );
14303
14304        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14305        let mut partial = runner_at(RunStatus::Reviewing);
14306        partial.state.repo = repo.clone();
14307        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14308        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14309            run: Some("exit 1".into()),
14310            ..Default::default()
14311        }];
14312        let selection = runner.state.pr_recovery.clone().unwrap();
14313        partial.state.pr_recovery = Some(selection.clone());
14314        partial.state.pr_head = Some(selection.branch.clone());
14315        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14316        let id = partial.state.id.clone();
14317        let error = match Runner::open_selected(
14318            &repo,
14319            partial.state,
14320            partial.roles,
14321            1,
14322            runner.state.base_commit.clone(),
14323            selection,
14324        )
14325        .await
14326        {
14327            Ok(_) => panic!("setup must fail"),
14328            Err(e) => e,
14329        };
14330        assert_eq!(
14331            error
14332                .downcast_ref::<crate::run::StartupFailure>()
14333                .unwrap()
14334                .run,
14335            id
14336        );
14337        let failed = RunState::load(&id).unwrap();
14338        assert_eq!(failed.status, RunStatus::Failed);
14339        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14340        assert!(Runner::resume(&id).is_err());
14341        runner.state.status = RunStatus::Ready;
14342        runner.state.driver_exited = true;
14343        runner.state.save().unwrap();
14344        let mut retry = runner_at(RunStatus::Reviewing);
14345        retry.state.repo = repo.clone();
14346        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14347        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14348        reselected.approved_at = jiff::Timestamp::now();
14349        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14350        retry.state.pr_head = Some(reselected.branch.clone());
14351        retry.state.pr_recovery = Some(reselected.clone());
14352        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14353            .await
14354            .unwrap();
14355        let restarted = Runner::open_selected(
14356            &repo,
14357            retry.state,
14358            retry.roles,
14359            1,
14360            runner.state.base_commit.clone(),
14361            reselected,
14362        )
14363        .await
14364        .unwrap();
14365        assert_eq!(
14366            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14367                .await
14368                .unwrap(),
14369            head
14370        );
14371        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14372        assert_eq!(
14373            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14374            head
14375        );
14376    }
14377
14378    #[tokio::test]
14379    #[cfg(unix)]
14380    async fn fallback_and_interrupted_restoration_require_coordination() {
14381        use std::os::fd::AsRawFd as _;
14382        ask_test_home();
14383        let tmp = tempfile::tempdir().unwrap();
14384        let mut runner = registered_runner(tmp.path()).await;
14385        runner.state.config.graph.timeout_resource_wait = 1;
14386        let winner = runner.state.candidates[0].clone();
14387        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14388        let leftover = winner.worktree.join("leftover");
14389        std::fs::write(&leftover, "untrusted edits").unwrap();
14390        let next = tmp.path().join("next-fixer");
14391        let mut failed = spec("first");
14392        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14393        let mut successor = spec("second");
14394        successor.command = vec![
14395            "sh".into(),
14396            "-c".into(),
14397            format!("touch '{}'; echo done", next.display()),
14398        ];
14399        let attempts = vec![
14400            (failed.clone(), "first".into()),
14401            (successor.clone(), "second".into()),
14402        ];
14403        let lock_path = runner.state.repo.join(".git/updater.lock");
14404        let external = std::fs::File::create(&lock_path).unwrap();
14405        assert_eq!(
14406            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14407            0
14408        );
14409        let artifacts = tmp.path().join("artifacts");
14410        let build = |spec, seat| SeatJob {
14411            spec,
14412            seat,
14413            cwd: winner.worktree.clone(),
14414            prompt: "fixture".into(),
14415            timeout: Duration::from_secs(10),
14416            allow_write: true,
14417            sessions: false,
14418            artifacts: artifacts.clone(),
14419            stem: "fix".into(),
14420            handover: None,
14421        };
14422        let (_, _, out) = runner
14423            .ask_fixer_among(
14424                attempts.clone(),
14425                "graft",
14426                None,
14427                Some((&winner.worktree, &head)),
14428                build,
14429            )
14430            .await;
14431        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14432        assert!(leftover.exists(), "contention must prevent clean");
14433        assert!(!next.exists());
14434        assert!(runner.state.handovers.is_empty());
14435        // Early interrupted-repair rollback uses the same protected route.
14436        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14437            from_run: "earlier".into(),
14438            generation: 1,
14439            mode: crate::recovery::Mode::VerifyRepair,
14440            reasons: Vec::new(),
14441            reviewed_head: None,
14442            rounds: Vec::new(),
14443            open_findings: Vec::new(),
14444            evidence: None,
14445            criteria_inherited: true,
14446            repair: crate::recovery::Repair::Started {
14447                before: head.clone(),
14448            },
14449        });
14450        assert!(
14451            runner
14452                .recovery_repair(&winner)
14453                .await
14454                .unwrap_err()
14455                .is::<crate::repo_lock::ResourceBlocked>()
14456        );
14457        assert!(matches!(
14458            runner.state.recovery.as_ref().unwrap().repair,
14459            crate::recovery::Repair::Started { .. }
14460        ));
14461        assert!(leftover.exists());
14462        drop(external);
14463        runner.recovery_repair(&winner).await.unwrap();
14464        assert!(!leftover.exists());
14465        assert!(matches!(
14466            runner.state.recovery.as_ref().unwrap().repair,
14467            crate::recovery::Repair::Done { .. }
14468        ));
14469        // A hard lock-open failure also leaves HEAD and untracked work intact.
14470        std::fs::remove_file(&lock_path).unwrap();
14471        std::fs::create_dir(&lock_path).unwrap();
14472        std::fs::write(&leftover, "untrusted edits").unwrap();
14473        let (_, _, out) = runner
14474            .ask_fixer_among(
14475                attempts,
14476                "graft",
14477                None,
14478                Some((&winner.worktree, &head)),
14479                build,
14480            )
14481            .await;
14482        assert!(matches!(out, AgentOutcome::Failed(_)));
14483        assert!(leftover.exists());
14484        assert!(!next.exists());
14485        assert_eq!(
14486            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14487            head
14488        );
14489    }
14490
14491    #[cfg(unix)]
14492    async fn registered_runner(root: &Path) -> Runner {
14493        repository_runner(root, crate::config::test_coordination()).await
14494    }
14495
14496    async fn repository_runner(
14497        root: &Path,
14498        coordination: Option<crate::config::Coordination>,
14499    ) -> Runner {
14500        ask_test_home();
14501        let repo = root.join("repo");
14502        std::fs::create_dir_all(&repo).unwrap();
14503        init_repo(&repo);
14504        if coordination.is_some() {
14505            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14506        }
14507        let mut runner = runner_at(RunStatus::Prep);
14508        runner.state.repo = repo.clone();
14509        runner.state.config.coordination = coordination;
14510        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14511        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14512        runner.state.config.graph.advise = false;
14513        runner.roles.implementers = vec![spec("implement")];
14514        runner.roles.judges = vec![spec("judge")];
14515        runner.prep().await.unwrap();
14516        runner
14517    }
14518
14519    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14520
14521    #[tokio::test]
14522    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14523        let tmp = tempfile::tempdir().unwrap();
14524        let mut runner = repository_runner(tmp.path(), None).await;
14525        let prompts = runner.state.config.prompts.clone();
14526        let run = runner.state.id.clone();
14527        let ctx = WaveCtx {
14528            carry_seats: false,
14529            pauses: None,
14530            run: &run,
14531            node: "advise",
14532            prompts: &prompts,
14533            cache: None,
14534            round: None,
14535        };
14536        let jobs: Vec<_> = ["alpha", "beta"]
14537            .into_iter()
14538            .map(|id| SeatJob {
14539                spec: scripted(id, AUTH_OUTAGE),
14540                seat: SeatState::new(id, id, 7),
14541                prompt: "propose".into(),
14542                cwd: runner.state.candidates[0].worktree.clone(),
14543                timeout: Duration::from_secs(10),
14544                allow_write: false,
14545                sessions: false,
14546                artifacts: agent::artifacts_dir(&runner.state.dir()),
14547                stem: id.into(),
14548                handover: None,
14549            })
14550            .collect();
14551        let mut losses = Vec::new();
14552        let results = ask_wave_with(
14553            jobs.clone(),
14554            runner.sem.clone(),
14555            2,
14556            &[],
14557            &ctx,
14558            &mut losses,
14559            &mut runner.state,
14560            &|_: &str| Ok(()),
14561        )
14562        .await;
14563        assert!(
14564            results
14565                .iter()
14566                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14567        );
14568        assert_eq!(runner.state.quota.len(), 2);
14569        assert!(
14570            runner
14571                .state
14572                .quota
14573                .iter()
14574                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14575        );
14576        assert!(!runner.state.advise_attempted);
14577        runner.stop_for_resource().unwrap();
14578        assert_eq!(runner.state.status, RunStatus::Blocked);
14579        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14580        assert_eq!(
14581            runner.state.resource_block.as_ref().unwrap().kind,
14582            "agent authentication"
14583        );
14584        // A machine loss followed by an ordinary failure is not evidence
14585        // that the whole node was unavailable, and must not earn a refund.
14586        runner.state.resource_block = None;
14587        runner.state.quota.clear();
14588        let mut mixed = jobs;
14589        mixed[1].spec.command = vec!["false".into()];
14590        let results = ask_wave_with(
14591            mixed,
14592            runner.sem.clone(),
14593            0,
14594            &[],
14595            &ctx,
14596            &mut losses,
14597            &mut runner.state,
14598            &|_: &str| Ok(()),
14599        )
14600        .await;
14601        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14602        assert!(runner.state.resource_block.is_none());
14603        assert_eq!(losses.len(), 1);
14604    }
14605
14606    #[tokio::test]
14607    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14608        let tmp = tempfile::tempdir().unwrap();
14609        let mut runner = repository_runner(tmp.path(), None).await;
14610        let implementer = scripted("implement", AUTH_OUTAGE);
14611        runner.roles.implementers = vec![implementer.clone()];
14612        runner.state.config.agents = vec![implementer];
14613        runner.implement().await.unwrap();
14614        assert_eq!(runner.state.status, RunStatus::Blocked);
14615        assert_eq!(runner.state.quota.len(), 1);
14616        assert_eq!(
14617            runner.state.quota[0].cause,
14618            agent::QuotaCause::Authentication
14619        );
14620        let candidate = &runner.state.candidates[0];
14621        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14622        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14623    }
14624
14625    #[tokio::test]
14626    #[cfg(unix)]
14627    async fn parked_implementer_persists_completed_work() {
14628        for registered in [false, true] {
14629            let tmp = tempfile::tempdir().unwrap();
14630            let mut runner = registered_runner(tmp.path()).await;
14631            if !registered {
14632                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14633            }
14634            let marker = tmp.path().join("started");
14635            let release = tmp.path().join("release");
14636            let mut implementer = spec("implement");
14637            implementer.command = vec![
14638                "sh".into(),
14639                "-c".into(),
14640                format!(
14641                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14642                    marker.display(),
14643                    release.display()
14644                ),
14645            ];
14646            runner.roles.implementers = vec![implementer.clone()];
14647            runner.state.config.agents = vec![implementer];
14648            let pause = runner.pause.clone();
14649            let (result, ()) = tokio::join!(runner.implement(), async {
14650                tokio::time::timeout(Duration::from_secs(10), async {
14651                    while !marker.exists() {
14652                        tokio::time::sleep(Duration::from_millis(10)).await;
14653                    }
14654                })
14655                .await
14656                .unwrap();
14657                pause.park();
14658                std::fs::write(&release, "go").unwrap();
14659            });
14660            result.unwrap();
14661            let candidate = &runner.state.candidates[0];
14662            assert!(candidate.commits > 0, "{candidate:?}");
14663            assert!(candidate.summary.contains("saved-result"));
14664            assert_eq!(candidate.files, 1);
14665            let before = candidate.commits;
14666            runner.implement().await.unwrap();
14667            assert_eq!(runner.state.candidates[0].commits, before);
14668        }
14669    }
14670
14671    #[tokio::test]
14672    #[cfg(unix)]
14673    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14674        let tmp = tempfile::tempdir().unwrap();
14675        let mut runner = registered_runner(tmp.path()).await;
14676        let winner = runner.state.candidates[0].clone();
14677        let cache = tmp.path().join("cache");
14678        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14679        runner.state.config.graph.timeout_resource_wait = 0;
14680        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14681        let crate::cache::AcquireOutcome::Acquired(held) =
14682            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14683        else {
14684            panic!("cache must be free")
14685        };
14686        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14687            from_run: "earlier".into(),
14688            generation: 1,
14689            mode: crate::recovery::Mode::VerifyRepair,
14690            reasons: Vec::new(),
14691            reviewed_head: None,
14692            rounds: Vec::new(),
14693            open_findings: Vec::new(),
14694            evidence: None,
14695            criteria_inherited: true,
14696            repair: crate::recovery::Repair::NotStarted,
14697        });
14698        runner.recovery_repair(&winner).await.unwrap();
14699        assert!(matches!(
14700            runner.state.recovery.as_ref().unwrap().repair,
14701            crate::recovery::Repair::WaitingForRestore { .. }
14702        ));
14703        drop(held);
14704        runner.state.resource_block = None;
14705        runner.state.config.graph.timeout_resource_wait = 1;
14706        runner.recovery_repair(&winner).await.unwrap();
14707        assert!(matches!(
14708            runner.state.recovery.as_ref().unwrap().repair,
14709            crate::recovery::Repair::Done { .. }
14710        ));
14711        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14712    }
14713
14714    #[tokio::test]
14715    #[cfg(unix)]
14716    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14717        use std::os::fd::AsRawFd as _;
14718        for graft in [false, true] {
14719            let tmp = tempfile::tempdir().unwrap();
14720            let mut runner = registered_runner(tmp.path()).await;
14721            runner.state.config.graph.timeout_resource_wait = 1;
14722            let winner = runner.state.candidates[0].clone();
14723            let marker = tmp.path().join("fixer-started");
14724            let mut fixer = spec("fixer");
14725            fixer.command = vec![
14726                "sh".into(),
14727                "-c".into(),
14728                format!("echo started >> '{}'; echo done", marker.display()),
14729            ];
14730            runner.roles.fixer = Some(vec![fixer]);
14731            if graft {
14732                let mut loser = winner.clone();
14733                loser.label = 'B';
14734                loser.index = 1;
14735                runner.state.candidates.push(loser);
14736                runner.state.config.graph.graft_runner_up = true;
14737                runner.state.tally = Some(Tally {
14738                    first_choice: BTreeMap::new(),
14739                    borda: BTreeMap::new(),
14740                    winner: winner.label,
14741                    rankings: 1,
14742                    unanimous_initial: true,
14743                    deliberated: false,
14744                    changed_votes: 0,
14745                    unanimous_final: true,
14746                    tie_break: None,
14747                    judges: 1,
14748                    present: 1,
14749                    quorum: 1,
14750                    met_quorum: true,
14751                    uncontested: None,
14752                });
14753                runner.state.judgements.push(crate::run::Judgement {
14754                    judge: 1,
14755                    seat: "judge-1".into(),
14756                    agent: "judge".into(),
14757                    ranking: vec![winner.label, 'B'],
14758                    reasons: BTreeMap::new(),
14759                    confidence: None,
14760                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14761                    order: vec![0, 1],
14762                    failed: None,
14763                    duration_ms: 0,
14764                });
14765            } else {
14766                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14767                    from_run: "earlier".into(),
14768                    generation: 1,
14769                    mode: crate::recovery::Mode::VerifyRepair,
14770                    reasons: Vec::new(),
14771                    reviewed_head: None,
14772                    rounds: Vec::new(),
14773                    open_findings: Vec::new(),
14774                    evidence: None,
14775                    criteria_inherited: true,
14776                    repair: crate::recovery::Repair::NotStarted,
14777                });
14778            }
14779            let reader = crate::repo_lock::SeatLease::for_checkout(
14780                &winner.worktree,
14781                false,
14782                crate::config::test_coordination().as_ref(),
14783            )
14784            .await
14785            .unwrap();
14786            let pause = runner.pause.clone();
14787            let (result, ()) = tokio::join!(
14788                async {
14789                    if graft {
14790                        runner.graft().await
14791                    } else {
14792                        runner.recovery_repair(&winner).await
14793                    }
14794                },
14795                async {
14796                    tokio::time::sleep(Duration::from_millis(100)).await;
14797                    pause.park();
14798                }
14799            );
14800            result.unwrap();
14801            assert!(!marker.exists(), "a blocked seat must not spawn");
14802            // Exercise the persisted state, as a binary replacement/resume does.
14803            runner.state = RunState::load(&runner.state.id).unwrap();
14804            if graft {
14805                assert_eq!(
14806                    runner.state.graft.as_ref().unwrap().outcome,
14807                    GraftOutcome::WaitingForRestore
14808                );
14809            } else {
14810                assert!(matches!(
14811                    runner.state.recovery.as_ref().unwrap().repair,
14812                    crate::recovery::Repair::WaitingForRestore { .. }
14813                ));
14814            }
14815            drop(reader);
14816            runner.pause = Pause::new();
14817            runner.state.resource_block = None;
14818            // Deferred restoration must still respect an external lock owner.
14819            let external =
14820                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14821            assert_eq!(
14822                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14823                0
14824            );
14825            let leftover = winner.worktree.join("leftover");
14826            std::fs::write(&leftover, "earlier fallback edits").unwrap();
14827            let result = if graft {
14828                runner.graft().await
14829            } else {
14830                runner.recovery_repair(&winner).await
14831            };
14832            assert!(
14833                result
14834                    .unwrap_err()
14835                    .is::<crate::repo_lock::ResourceBlocked>()
14836            );
14837            assert!(leftover.exists());
14838            assert!(!marker.exists());
14839            drop(external);
14840            runner.state.resource_block = None;
14841            let result = if graft {
14842                runner.graft().await
14843            } else {
14844                runner.recovery_repair(&winner).await
14845            };
14846            result.unwrap();
14847            assert!(!leftover.exists(), "restore must precede retry");
14848            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14849            if graft {
14850                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14851            } else {
14852                assert!(matches!(
14853                    runner.state.recovery.as_ref().unwrap().repair,
14854                    crate::recovery::Repair::Done { .. }
14855                ));
14856            }
14857        }
14858    }
14859
14860    #[tokio::test]
14861    #[cfg(unix)]
14862    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14863        use std::os::fd::AsRawFd as _;
14864        let tmp = tempfile::tempdir().unwrap();
14865        let mut runner = registered_runner(tmp.path()).await;
14866        runner.state.config.graph.timeout_resource_wait = 1;
14867        let winner = runner.state.candidates[0].clone();
14868        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14869        runner.state.tally = Some(Tally {
14870            first_choice: BTreeMap::new(),
14871            borda: BTreeMap::new(),
14872            winner: winner.label,
14873            rankings: 0,
14874            unanimous_initial: false,
14875            deliberated: false,
14876            changed_votes: 0,
14877            unanimous_final: false,
14878            tie_break: None,
14879            judges: 0,
14880            present: 0,
14881            quorum: 0,
14882            met_quorum: true,
14883            uncontested: Some("fixture".into()),
14884        });
14885        runner.state.graft = Some(GraftRecord {
14886            pre: pre.clone(),
14887            post: None,
14888            lines: None,
14889            limit: None,
14890            agent: None,
14891            outcome: GraftOutcome::Running,
14892            requested: Vec::new(),
14893            adoption: None,
14894            adoption_error: None,
14895        });
14896        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14897        git::commit_all(&winner.worktree, "unfinished graft")
14898            .await
14899            .unwrap();
14900        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14901        let leftover = winner.worktree.join("leftover");
14902        std::fs::write(&leftover, "untrusted edits").unwrap();
14903        let path = runner.state.repo.join(".git/updater.lock");
14904        let external = std::fs::File::create(&path).unwrap();
14905        assert_eq!(
14906            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14907            0
14908        );
14909        assert!(
14910            runner
14911                .graft()
14912                .await
14913                .unwrap_err()
14914                .is::<crate::repo_lock::ResourceBlocked>()
14915        );
14916        assert!(matches!(
14917            runner.state.graft.as_ref().unwrap().outcome,
14918            GraftOutcome::Running
14919        ));
14920        assert_eq!(
14921            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14922            unfinished
14923        );
14924        assert!(leftover.exists());
14925        drop(external);
14926        std::fs::remove_file(&path).unwrap();
14927        std::fs::create_dir(&path).unwrap();
14928        assert!(runner.graft().await.is_err());
14929        assert!(matches!(
14930            runner.state.graft.as_ref().unwrap().outcome,
14931            GraftOutcome::Running
14932        ));
14933        assert_eq!(
14934            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14935            unfinished
14936        );
14937        assert!(leftover.exists());
14938        std::fs::remove_dir(&path).unwrap();
14939        runner.graft().await.unwrap();
14940        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14941        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
14942        assert!(!leftover.exists());
14943    }
14944
14945    #[tokio::test]
14946    #[cfg(unix)]
14947    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
14948        use std::os::fd::AsRawFd as _;
14949        let tmp = tempfile::tempdir().unwrap();
14950        let mut runner = registered_runner(tmp.path()).await;
14951        runner.state.prep_complete = Some(false);
14952        runner.state.config.graph.timeout_resource_wait = 1;
14953        runner.state.save().unwrap();
14954        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14955        assert_eq!(
14956            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14957            0
14958        );
14959        runner.execute().await.unwrap();
14960        let disk = RunState::load(&runner.state.id).unwrap();
14961        assert!(disk.driver_exited);
14962        assert_eq!(disk.driver_pid, None);
14963        assert_eq!(runner.state.status, RunStatus::Blocked);
14964        assert_eq!(
14965            runner.state.resource_block.as_ref().unwrap().kind,
14966            "repository coordination"
14967        );
14968        assert!(runner.state.prep_failure.is_none());
14969        assert!(runner.state.seats.is_empty());
14970        runner.pause.park();
14971        // Include headroom for Git subprocesses under parallel test load;
14972        // the repository contention deadline itself remains one second.
14973        tokio::time::timeout(Duration::from_secs(10), runner.execute())
14974            .await
14975            .unwrap()
14976            .unwrap();
14977        assert!(runner.state.parked);
14978        assert!(runner.state.prep_failure.is_none());
14979        let disk = RunState::load(&runner.state.id).unwrap();
14980        assert!(disk.driver_exited);
14981        assert_eq!(disk.driver_pid, None);
14982    }
14983
14984    #[tokio::test]
14985    #[cfg(unix)]
14986    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14987        use std::os::fd::AsRawFd as _;
14988        let tmp = tempfile::tempdir().unwrap();
14989        let mut runner = registered_runner(tmp.path()).await;
14990        runner.state.prep_complete = Some(false);
14991        runner.state.config.graph.timeout_resource_wait = 30;
14992        runner.state.save().unwrap();
14993        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14994        assert_eq!(
14995            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14996            0
14997        );
14998        assert!(
14999            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15000                .await
15001                .is_err()
15002        );
15003        let disk = RunState::load(&runner.state.id).unwrap();
15004        assert!(disk.driver_exited);
15005        assert_eq!(disk.driver_pid, None);
15006        assert!(disk.status.resumable());
15007        assert!(runner.state.driver_exited);
15008        assert_eq!(runner.state.driver_pid, None);
15009        runner.state.save().unwrap();
15010        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15011    }
15012
15013    #[tokio::test]
15014    #[cfg(unix)]
15015    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15016        let tmp = tempfile::tempdir().unwrap();
15017        let mut runner = registered_runner(tmp.path()).await;
15018        let winner = runner.state.candidates[0].clone();
15019        let cache = tmp.path().join("cache");
15020        let started = tmp.path().join("pre-gate-started");
15021        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15022        runner.state.config.verify.gate =
15023            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15024        let home = crate::run::home();
15025        let owner =
15026            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15027        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15028            crate::cache::AcquireOutcome::Acquired(g) => g,
15029            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15030        };
15031        let mut gate = Box::pin(runner.run_gate(&winner));
15032        assert!(
15033            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15034                .await
15035                .is_err()
15036        );
15037        assert!(!started.exists(), "pre-gate must wait for the cache too");
15038        // Simulate the wave that already owns the cache entering invoke.
15039        // It must obtain the editor reservation and finish without waiting
15040        // for the gate that is queued behind its cache.
15041        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15042        let editor = crate::repo_lock::SeatLease::checkout_with(
15043            &winner.worktree,
15044            true,
15045            crate::config::test_coordination().as_ref(),
15046            &wait,
15047        )
15048        .await
15049        .unwrap();
15050        drop(editor);
15051        guard.release();
15052        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15053            .await
15054            .unwrap()
15055            .unwrap();
15056        drop(gate);
15057        assert!(outcomes.iter().all(CommandOutcome::ok));
15058        assert!(started.exists());
15059    }
15060
15061    #[tokio::test]
15062    #[cfg(unix)]
15063    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15064        use std::os::fd::AsRawFd as _;
15065
15066        ask_test_home();
15067        let tmp = tempfile::tempdir().unwrap();
15068        let repo = tmp.path().join("repo");
15069        std::fs::create_dir_all(&repo).unwrap();
15070        init_repo(&repo);
15071        let common = repo.join(".git");
15072        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15073        let mut runner = runner_at(RunStatus::Prep);
15074        runner.state.repo = repo.clone();
15075        runner.state.config.coordination = crate::config::test_coordination();
15076        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15077        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15078        runner.state.config.graph.advise = false;
15079        runner.roles.implementers = vec![spec("implement")];
15080        runner.roles.judges = vec![spec("judge")];
15081        runner.prep().await.unwrap();
15082        let winner = runner.state.candidates[0].clone();
15083        // Emulate edits already left by a completed seat. The fixture CLI
15084        // only reports completion; it performs no repository operations.
15085        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15086        let completed = tmp.path().join("seat-completed");
15087        runner.roles.implementers[0].command = vec![
15088            "sh".into(),
15089            "-c".into(),
15090            format!("touch '{}'; echo completed", completed.display()),
15091        ];
15092        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15093        assert_eq!(
15094            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15095            0
15096        );
15097        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15098        let mut implement = Box::pin(runner.implement());
15099        tokio::select! {
15100            result = &mut implement => panic!("rescue must wait: {result:?}"),
15101            result = tokio::time::timeout(Duration::from_secs(10), async {
15102                while !completed.exists() {
15103                    tokio::time::sleep(Duration::from_millis(20)).await;
15104                }
15105            }) => result.unwrap(),
15106        }
15107        assert!(
15108            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15109                .await
15110                .is_err()
15111        );
15112        assert_eq!(
15113            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15114            before
15115        );
15116        assert_eq!(
15117            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15118            0
15119        );
15120        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15121            .await
15122            .unwrap()
15123            .unwrap();
15124        drop(implement);
15125        assert_ne!(
15126            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15127            before
15128        );
15129        assert!(git::is_clean(&winner.worktree).await.unwrap());
15130
15131        let started = tmp.path().join("pre-gate-started");
15132        let proceed = tmp.path().join("pre-gate-proceed");
15133        let checked = tmp.path().join("gate-ran");
15134        runner.state.config.verify.pre_gate = vec![format!(
15135            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15136            started.display(),
15137            proceed.display()
15138        )];
15139        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15140        assert_eq!(
15141            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15142            0
15143        );
15144        let mut gate = Box::pin(runner.run_gate(&winner));
15145        assert!(
15146            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15147                .await
15148                .is_err()
15149        );
15150        assert!(!started.exists());
15151        assert!(!checked.exists());
15152        assert!(git::is_clean(&winner.worktree).await.unwrap());
15153        assert_eq!(
15154            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15155            0
15156        );
15157        tokio::select! {
15158            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15159            result = tokio::time::timeout(Duration::from_secs(10), async {
15160                while !started.exists() {
15161                    tokio::time::sleep(Duration::from_millis(20)).await;
15162                }
15163            }) => result.unwrap(),
15164        }
15165        assert_eq!(
15166            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15167            -1,
15168            "mechanical edits must hold the real lock"
15169        );
15170        std::fs::write(proceed, "ready").unwrap();
15171        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15172            .await
15173            .unwrap()
15174            .unwrap();
15175        drop(gate);
15176        assert!(outcomes.iter().all(CommandOutcome::ok));
15177        assert!(checked.exists());
15178        assert!(runner.state.pre_gate_commit.is_some());
15179        assert!(git::is_clean(&winner.worktree).await.unwrap());
15180        assert_eq!(
15181            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15182            0,
15183            "gate completion must release coordination"
15184        );
15185        drop(outsider);
15186        std::fs::remove_file(common.join("updater.lock")).unwrap();
15187        std::fs::create_dir(common.join("updater.lock")).unwrap();
15188        std::fs::remove_file(&started).unwrap();
15189        std::fs::remove_file(&checked).unwrap();
15190        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15191        let error = runner.run_gate(&winner).await.unwrap_err();
15192        assert!(
15193            error
15194                .to_string()
15195                .contains("acquire mandatory repository lock")
15196        );
15197        assert!(
15198            !started.exists(),
15199            "failed acquisition must prevent pre-gate"
15200        );
15201        assert!(!checked.exists(), "failed acquisition must prevent gate");
15202        assert!(git::is_clean(&winner.worktree).await.unwrap());
15203        assert_eq!(
15204            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15205            head
15206        );
15207    }
15208
15209    #[tokio::test]
15210    #[cfg(unix)]
15211    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15212        use std::os::fd::AsRawFd as _;
15213        use std::time::Duration;
15214
15215        ask_test_home();
15216        let tmp = tempfile::tempdir().unwrap();
15217        let repo = tmp.path().join("repo");
15218        std::fs::create_dir_all(&repo).unwrap();
15219        init_repo(&repo);
15220        let common = repo.join(".git");
15221        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15222        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15223        assert_eq!(
15224            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15225            0
15226        );
15227        let mut runner = runner_at(RunStatus::Prep);
15228        runner.state.repo = repo.clone();
15229        runner.state.config.coordination = crate::config::test_coordination();
15230        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15231        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15232        runner.state.config.graph.advise = true;
15233        runner.state.config.graph.advisors = 1;
15234        runner.roles.implementers = vec![spec("implement")];
15235        runner.roles.judges = vec![spec("judge")];
15236        let root = runner.state.worktree_root();
15237        let started = tmp.path().join("setup-started");
15238        let proceed = tmp.path().join("setup-proceed");
15239        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15240            run: Some(format!(
15241                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15242                started.display(),
15243                proceed.display()
15244            )),
15245            ..Default::default()
15246        }];
15247        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15248        let config = git::git(&repo, &["config", "--local", "--list"])
15249            .await
15250            .unwrap();
15251        let mut prep = Box::pin(runner.prep());
15252        assert!(
15253            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15254                .await
15255                .is_err()
15256        );
15257        assert!(
15258            !root.exists(),
15259            "contention must prevent even checkout creation"
15260        );
15261        assert!(!started.exists(), "contention must prevent setup");
15262        assert_eq!(
15263            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15264            branches
15265        );
15266        assert_eq!(
15267            git::git(&repo, &["config", "--local", "--list"])
15268                .await
15269                .unwrap(),
15270            config
15271        );
15272        drop(lock);
15273        tokio::select! {
15274            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15275            result = tokio::time::timeout(Duration::from_secs(10), async {
15276                while !started.exists() {
15277                    tokio::time::sleep(Duration::from_millis(20)).await;
15278                }
15279            }) => result.unwrap(),
15280        }
15281        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15282        assert_eq!(
15283            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15284            -1
15285        );
15286        assert!(
15287            tokio::time::timeout(
15288                Duration::from_millis(100),
15289                crate::repo_lock::SeatLease::for_checkout(
15290                    &repo,
15291                    true,
15292                    crate::config::test_coordination().as_ref()
15293                )
15294            )
15295            .await
15296            .is_err(),
15297            "preparation must also exclude local editors"
15298        );
15299        std::fs::write(proceed, "ready").unwrap();
15300        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15301            .await
15302            .unwrap()
15303            .unwrap();
15304        drop(prep);
15305        assert_eq!(runner.state.prep_complete, Some(true));
15306        assert!(root.join("judge-1").exists());
15307        assert!(root.join("advisor-1").exists());
15308        assert_eq!(
15309            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15310            0
15311        );
15312    }
15313
15314    #[tokio::test]
15315    #[cfg(unix)]
15316    async fn preparation_lock_failure_prevents_repository_changes() {
15317        ask_test_home();
15318        let tmp = tempfile::tempdir().unwrap();
15319        let repo = tmp.path().join("repo");
15320        std::fs::create_dir_all(&repo).unwrap();
15321        init_repo(&repo);
15322        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15323        // A directory at the required lock path makes opening the real lock
15324        // fail, rather than merely exercising an invalid registration.
15325        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15326        let mut runner = runner_at(RunStatus::Prep);
15327        runner.state.repo = repo.clone();
15328        runner.state.config.coordination = crate::config::test_coordination();
15329        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15330        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15331        runner.roles.implementers = vec![spec("implement")];
15332        let error = runner.prep().await.unwrap_err();
15333        assert!(
15334            error
15335                .to_string()
15336                .contains("acquire mandatory repository lock")
15337        );
15338        assert!(!runner.state.worktree_root().exists());
15339        assert!(runner.state.candidates.is_empty());
15340        assert!(!repo.join(".git/config.worktree").exists());
15341        assert_eq!(
15342            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15343            "* main"
15344        );
15345    }
15346
15347    #[tokio::test]
15348    async fn preparation_records_creation_intent_before_git_can_fail() {
15349        ask_test_home();
15350        let tmp = tempfile::tempdir().unwrap();
15351        let repo = tmp.path().join("repo");
15352        std::fs::create_dir_all(&repo).unwrap();
15353        init_repo(&repo);
15354        let mut runner = runner_at(RunStatus::Prep);
15355        runner.state.repo = repo.clone();
15356        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15357        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15358        runner.state.config.blind.commit_msg_hook = false;
15359        runner.roles.implementers = vec![spec("one")];
15360        let label = blind::assign_labels(1, runner.state.seed)[0];
15361        let branch = runner.state.branch_for(label);
15362        // A conflicting branch makes Git refuse creation before a candidate
15363        // can be registered. The durable intent must still identify it.
15364        git::git(&repo, &["branch", &branch]).await.unwrap();
15365        assert!(runner.prep().await.is_err());
15366        let saved = RunState::load(&runner.state.id).unwrap();
15367        assert!(saved.candidates.is_empty());
15368        let event = saved
15369            .events
15370            .iter()
15371            .find(|e| e.node == "prep-intent")
15372            .unwrap();
15373        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15374        assert_eq!(intent["branch"], branch);
15375        assert_eq!(intent["start"], runner.state.base_commit);
15376        assert_eq!(
15377            intent["worktree"],
15378            runner
15379                .state
15380                .worktree_root()
15381                .join(format!("cand-{label}"))
15382                .to_string_lossy()
15383                .as_ref()
15384        );
15385        assert_eq!(intent["picks"], serde_json::json!([]));
15386    }
15387
15388    #[tokio::test]
15389    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15390        ask_test_home();
15391        let tmp = tempfile::tempdir().unwrap();
15392        let repo = tmp.path().join("repo");
15393        std::fs::create_dir_all(&repo).unwrap();
15394        init_repo(&repo);
15395        let mut runner = runner_at(RunStatus::Prep);
15396        runner.state.repo = repo.clone();
15397        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15398        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15399        runner.state.sources.pick_commits.push(refs::Input {
15400            token: "missing".into(),
15401            sha: "0".repeat(40),
15402            provenance: refs::Provenance::ExplicitCli,
15403        });
15404        assert!(runner.prep().await.is_err());
15405        assert!(runner.state.candidates.is_empty());
15406        assert!(!runner.state.worktree_root().exists());
15407    }
15408
15409    #[tokio::test]
15410    async fn prose_and_legacy_inference_never_import_into_candidates() {
15411        ask_test_home();
15412        let tmp = tempfile::tempdir().unwrap();
15413        let repo = tmp.path().join("repo");
15414        std::fs::create_dir_all(&repo).unwrap();
15415        init_repo(&repo);
15416        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15417        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15418            .await
15419            .unwrap();
15420        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15421        git::git(&repo, &["add", "."]).await.unwrap();
15422        git::git(&repo, &["commit", "-m", "historical implementation"])
15423            .await
15424            .unwrap();
15425        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15426        git::git(&repo, &["checkout", "main"]).await.unwrap();
15427        for instruction in [
15428            format!("Example: magi/history/A at {source}"),
15429            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15430            format!("Never import magi/history/A or {source}"),
15431            format!("Conversation answer: I mentioned {source} only as background"),
15432        ] {
15433            let mut runner = runner_at(RunStatus::Prep);
15434            runner.state.repo = repo.clone();
15435            runner.state.base_commit = base.clone();
15436            runner.state.instruction = instruction.clone();
15437            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15438            runner.state.config.blind.commit_msg_hook = false;
15439            runner.state.seeds = vec![refs::Seed {
15440                token: "magi/history/A".into(),
15441                sha: source.clone(),
15442                kind: refs::SeedKind::Unmerged,
15443                branch: true,
15444                detail: String::new(),
15445            }];
15446            runner.roles.implementers = vec![spec("one")];
15447            assert_eq!(seeded_instruction(&runner.state), instruction);
15448            // Model an old record with inferred seeds but no structured input field.
15449            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15450            legacy.as_object_mut().unwrap().remove("sources");
15451            runner.state = serde_json::from_value(legacy).unwrap();
15452            assert_eq!(runner.state.sources, refs::Sources::default());
15453            runner.state.save().unwrap();
15454            runner.state = RunState::load(&runner.state.id).unwrap();
15455            runner.prep().await.unwrap();
15456            for candidate in &runner.state.candidates {
15457                assert_eq!(
15458                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15459                    base
15460                );
15461                assert!(!candidate.worktree.join("unwanted").exists());
15462            }
15463        }
15464    }
15465
15466    #[tokio::test]
15467    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15468        ask_test_home();
15469        let tmp = tempfile::tempdir().unwrap();
15470        let repo = tmp.path().join("repo");
15471        std::fs::create_dir_all(&repo).unwrap();
15472        init_repo(&repo);
15473        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15474        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15475        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15476        git::git(&repo, &["add", "."]).await.unwrap();
15477        git::git(&repo, &["commit", "-m", "required seed"])
15478            .await
15479            .unwrap();
15480        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15481        git::git(&repo, &["checkout", "main"]).await.unwrap();
15482        let mut runner = runner_at(RunStatus::Prep);
15483        runner.state.repo = repo.clone();
15484        runner.state.base_commit = base.clone();
15485        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15486        runner.state.config.graph.advise = false;
15487        runner.state.config.blind.commit_msg_hook = false;
15488        runner.state.sources =
15489            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15490                .await
15491                .unwrap();
15492        runner.roles.implementers = vec![spec("one"), spec("two")];
15493        runner.roles.judges = vec![spec("judge")];
15494        let label = blind::assign_labels(2, runner.state.seed)[0];
15495        let branch = runner.state.branch_for(label);
15496        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15497        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15498            .await
15499            .unwrap();
15500        runner.state.candidates.push(Candidate {
15501            index: 0,
15502            label,
15503            agent: "one".into(),
15504            branch,
15505            worktree,
15506            summary: String::new(),
15507            stat: String::new(),
15508            files: 0,
15509            commits: 0,
15510            empty: false,
15511            failed: None,
15512            verified_noop: None,
15513            duration_ms: 0,
15514            folded: false,
15515        });
15516        runner.state.save().unwrap();
15517        // Reload the checkpoint immediately before its first cherry-pick.
15518        runner.state = RunState::load(&runner.state.id).unwrap();
15519        assert_eq!(runner.state.prep_complete, Some(false));
15520        runner.prep().await.unwrap();
15521        assert_eq!(runner.state.prep_complete, Some(true));
15522        assert_eq!(runner.state.candidates.len(), 2);
15523        for candidate in &runner.state.candidates {
15524            assert_eq!(
15525                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15526                git::tree_of(&repo, &seed).await.unwrap()
15527            );
15528            assert_eq!(
15529                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15530                "required commit"
15531            );
15532        }
15533        assert!(runner.state.worktree_root().join("judge-1").exists());
15534        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15535        runner.prep().await.unwrap();
15536        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15537
15538        runner.state.prep_complete = Some(false);
15539        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15540            run: Some(format!(
15541                "test -f '{}'",
15542                tmp.path().join("dependency-ready").display()
15543            )),
15544            ..Default::default()
15545        }];
15546        assert!(runner.prep().await.is_err());
15547        assert!(runner.state.prep_failure.is_some());
15548        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15549        runner.state = RunState::load(&runner.state.id).unwrap();
15550        assert!(
15551            runner.prep().await.is_err(),
15552            "correcting the cause alone must not auto-retry"
15553        );
15554        let failed_at = runner.state.events.last().unwrap().at;
15555        assert!(
15556            runner.retry_preparation(failed_at).is_err(),
15557            "old approvals cannot unlock the hold"
15558        );
15559        let mut task = crate::queue::Task::new(
15560            "retry".into(),
15561            "retry setup".into(),
15562            repo.clone(),
15563            crate::queue::Source::Human,
15564        );
15565        task.start(runner.state.id.clone());
15566        task.hold_machine(Some("setup failed".into()));
15567        task.release_operator("cli", None);
15568        runner
15569            .retry_preparation(task.resumption.as_ref().unwrap().at)
15570            .unwrap();
15571        runner.prep().await.unwrap();
15572        assert!(runner.state.prep_failure.is_none());
15573        assert_eq!(runner.state.prep_complete, Some(true));
15574        assert_eq!(runner.state.candidates.len(), 2);
15575        assert!(runner.state.worktree_root().join("judge-1").exists());
15576        for candidate in &runner.state.candidates {
15577            assert_eq!(
15578                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15579                git::tree_of(&repo, &seed).await.unwrap()
15580            );
15581        }
15582    }
15583
15584    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15585        let mut heads = Vec::new();
15586        for candidate in candidates {
15587            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15588        }
15589        heads
15590    }
15591
15592    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15593        ask_test_home();
15594        let tmp = tempfile::tempdir().unwrap();
15595        let repo = tmp.path().join("repo");
15596        std::fs::create_dir(&repo).unwrap();
15597        init_repo(&repo);
15598        let mut runner = runner_at(RunStatus::Reviewing);
15599        runner.state.repo = repo.clone();
15600        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15601        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15602        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15603            let branch = format!("candidate-{label}");
15604            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15605            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15606                .await
15607                .unwrap();
15608            if label == 'B' {
15609                std::fs::write(worktree.join("change"), "winner").unwrap();
15610                git::rescue_commit(&worktree, "candidate change")
15611                    .await
15612                    .unwrap();
15613            }
15614            runner.state.candidates.push(Candidate {
15615                index,
15616                label,
15617                agent: "alpha".into(),
15618                branch,
15619                worktree,
15620                summary: "completed reply".into(),
15621                stat: String::new(),
15622                files: usize::from(label == 'B'),
15623                commits: usize::from(label == 'B'),
15624                empty: label == 'A',
15625                failed: None,
15626                verified_noop: None,
15627                duration_ms: 1,
15628                folded: false,
15629            });
15630        }
15631        runner.state.judge_skipped = true;
15632        runner.state.tally = Some(Tally {
15633            first_choice: BTreeMap::new(),
15634            borda: BTreeMap::new(),
15635            winner: 'B',
15636            rankings: 0,
15637            unanimous_initial: false,
15638            deliberated: false,
15639            changed_votes: 0,
15640            unanimous_final: false,
15641            tie_break: None,
15642            judges: 0,
15643            present: 0,
15644            quorum: 0,
15645            met_quorum: true,
15646            uncontested: Some("only B produced a change".into()),
15647        });
15648        runner.state.save().unwrap();
15649        (tmp, runner)
15650    }
15651
15652    #[tokio::test]
15653    #[cfg(unix)]
15654    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15655        for (solo, failed_checkpoint) in
15656            [(true, false), (false, false), (true, true), (false, true)]
15657        {
15658            let (_tmp, mut runner) = preservation_resume_fixture().await;
15659            if solo {
15660                runner.state.candidates.retain(|c| c.label == 'A');
15661            }
15662            runner.state.candidates[0].empty = false;
15663            runner.state.candidates[0].summary.clear();
15664            runner.state.tally = None;
15665            runner.state.judge_skipped = false;
15666            runner.state.status = RunStatus::Implementing;
15667            runner.state.config.disk.min_free_bytes = u64::MAX;
15668            let mut implementer = spec("alpha");
15669            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15670            runner.roles.implementers = vec![implementer];
15671            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15672            if failed_checkpoint {
15673                std::fs::create_dir_all(&patch).unwrap();
15674            }
15675            runner.state.save().unwrap();
15676            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15677            runner.state = RunState::load(&runner.state.id).unwrap();
15678            assert_eq!(runner.state.status, RunStatus::Blocked);
15679            assert!(runner.state.interrupted_implementations.contains(&'A'));
15680            assert_eq!(runner.state.candidates[0].commits, 0);
15681            assert!(!runner.state.candidates[0].empty);
15682            assert!(runner.state.candidates[0].failed.is_none());
15683            if failed_checkpoint {
15684                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15685                std::fs::remove_dir(patch).unwrap();
15686            }
15687            runner.state.resource_block = None;
15688            runner.state.config.disk.min_free_bytes = 0;
15689            runner.roles.implementers[0].command = vec![
15690                "sh".into(),
15691                "-c".into(),
15692                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15693            ];
15694            runner.implement().await.unwrap();
15695            assert!(runner.state.interrupted_implementations.is_empty());
15696            assert!(runner.state.candidate_preservation.is_empty());
15697            assert_eq!(runner.state.candidates[0].commits, 1);
15698            assert!(!runner.state.candidates[0].empty);
15699            assert!(
15700                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15701                    .unwrap()
15702                    .contains("retried empty interrupted seat")
15703            );
15704            if !solo {
15705                assert_eq!(runner.state.candidates[1].commits, 1);
15706            }
15707        }
15708    }
15709
15710    #[tokio::test]
15711    #[cfg(unix)]
15712    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15713        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15714            let (_tmp, mut runner) = preservation_resume_fixture().await;
15715            runner.state.candidates.retain(|c| c.label == 'A');
15716            runner.state.candidates[0].empty = false;
15717            runner.state.candidates[0].summary.clear();
15718            runner.state.tally = None;
15719            runner.state.judge_skipped = false;
15720            runner.state.status = RunStatus::Implementing;
15721            // Force the real post-spawn watchdog, independently of available
15722            // disk space. Dirty/committed work predates its first measurement.
15723            runner.state.config.disk.min_free_bytes = u64::MAX;
15724            let wt = runner.state.candidates[0].worktree.clone();
15725            std::fs::write(
15726                wt.join("interrupted-source"),
15727                "retain implementation work\n",
15728            )
15729            .unwrap();
15730            if committed {
15731                git::rescue_commit(&wt, "implementation checkpoint")
15732                    .await
15733                    .unwrap();
15734            }
15735            let lock = git::git(
15736                &wt,
15737                &[
15738                    "rev-parse",
15739                    "--path-format=absolute",
15740                    "--git-path",
15741                    "index.lock",
15742                ],
15743            )
15744            .await
15745            .unwrap();
15746            if block_rescue {
15747                std::fs::write(&lock, "fixture-owned lock").unwrap();
15748            }
15749            let mut implementer = spec("alpha");
15750            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15751            runner.roles.implementers = vec![implementer];
15752            runner.state.save().unwrap();
15753            let result = runner.implement().await;
15754            assert_eq!(result.is_err(), block_rescue);
15755            assert_eq!(
15756                runner.state.resource_block.as_ref().unwrap().kind,
15757                "disk pressure"
15758            );
15759            assert!(
15760                runner
15761                    .state
15762                    .events
15763                    .iter()
15764                    .any(|e| e.message.contains("agent execution interrupted"))
15765            );
15766            assert!(
15767                runner
15768                    .state
15769                    .dir()
15770                    .join("artifacts/candidate-A-recovery/staged.patch")
15771                    .exists()
15772            );
15773            if block_rescue {
15774                let saved = RunState::load(&runner.state.id).unwrap();
15775                assert!(saved.candidate_preservation.contains_key(&'A'));
15776                assert!(!saved.candidates[0].empty);
15777                assert_eq!(
15778                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15779                    "retain implementation work\n"
15780                );
15781                std::fs::remove_file(lock).unwrap();
15782                runner.state = saved;
15783            } else {
15784                assert_eq!(runner.state.status, RunStatus::Blocked);
15785                assert_eq!(runner.state.candidates[0].commits, 1);
15786            }
15787            // Recovery must save the candidate before deciding whether another
15788            // implementation is needed. No implementer is available on retry.
15789            runner.roles.implementers.clear();
15790            runner.state.config.disk.min_free_bytes = 0;
15791            runner.state.resource_block = None;
15792            runner.implement().await.unwrap();
15793            assert!(runner.state.candidate_preservation.is_empty());
15794            assert_eq!(runner.state.candidates[0].commits, 1);
15795            assert!(!runner.state.candidates[0].empty);
15796            assert!(git::is_clean(&wt).await.unwrap());
15797            assert!(
15798                crate::run::read_artifact(&runner.state, "cand-A.patch")
15799                    .unwrap()
15800                    .contains("retain implementation work")
15801            );
15802        }
15803    }
15804
15805    #[tokio::test]
15806    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15807        let (_tmp, mut runner) = preservation_resume_fixture().await;
15808        // Schema 15 already confirmed the unfolded no-change result.
15809        runner.implement().await.unwrap();
15810        assert!(runner.state.candidate_preservation.is_empty());
15811        assert!(
15812            !runner
15813                .state
15814                .dir()
15815                .join("artifacts/candidate-A-recovery")
15816                .exists()
15817        );
15818        runner.fold_losers().await.unwrap();
15819        assert!(!runner.state.candidates[0].worktree.exists());
15820        for schema in [14, crate::run::SCHEMA] {
15821            runner.state.schema = schema;
15822            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15823            runner.state.save().unwrap();
15824            runner.state = RunState::load(&runner.state.id).unwrap();
15825            runner.implement().await.unwrap();
15826            assert!(runner.state.candidate_preservation.is_empty());
15827            assert!(runner.state.candidates[0].folded);
15828            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15829        }
15830    }
15831
15832    #[tokio::test]
15833    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15834        let (_tmp, mut runner) = preservation_resume_fixture().await;
15835        runner.state.candidates.retain(|c| c.label == 'A');
15836        runner.state.tally = None;
15837        runner.state.judge_skipped = false;
15838        let candidate = &mut runner.state.candidates[0];
15839        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15840        candidate.failed = Some("deployment validation unavailable".into());
15841        std::fs::write(
15842            candidate.worktree.join("repair"),
15843            "future launcher policy\n",
15844        )
15845        .unwrap();
15846        runner
15847            .state
15848            .candidate_preservation
15849            .insert('A', "awaiting persistence".into());
15850        runner.persist_candidates().await.unwrap();
15851        let candidate = &runner.state.candidates[0];
15852        assert!(!candidate.empty);
15853        assert_eq!(candidate.commits, 1);
15854        assert_eq!(candidate.files, 1);
15855        assert!(candidate.failed.is_none());
15856        assert!(candidate.summary.contains("EPERM"));
15857        assert!(
15858            crate::run::read_artifact(&runner.state, "cand-A.patch")
15859                .unwrap()
15860                .contains("future launcher policy")
15861        );
15862        assert!(runner.state.candidate_preservation.is_empty());
15863    }
15864
15865    #[tokio::test]
15866    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15867        let (_tmp, mut runner) = preservation_resume_fixture().await;
15868        let wt = runner.state.candidates[0].worktree.clone();
15869        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15870        runner.state.schema = 14;
15871        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15872        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15873        let tally = serde_json::to_value(&runner.state.tally).unwrap();
15874        for _ in 0..2 {
15875            assert!(runner.implement().await.is_err());
15876            assert_eq!(runner.state.status, RunStatus::Blocked);
15877            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15878            assert!(
15879                runner.state.candidates[0].empty,
15880                "old verdict must remain unusable"
15881            );
15882            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15883            assert!(runner.state.judge_skipped);
15884            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15885            assert!(
15886                crate::run::read_artifact(&runner.state, "cand-A.patch")
15887                    .unwrap()
15888                    .contains("owned edits")
15889            );
15890            assert!(wt.exists());
15891            assert_eq!(
15892                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15893                    .await
15894                    .unwrap(),
15895                1
15896            );
15897            runner.state = RunState::load(&runner.state.id).unwrap();
15898        }
15899    }
15900
15901    #[tokio::test]
15902    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15903        let (_tmp, mut runner) = preservation_resume_fixture().await;
15904        let wt = runner.state.candidates[0].worktree.clone();
15905        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15906        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15907        let index_lock = PathBuf::from(
15908            git::git(
15909                &wt,
15910                &[
15911                    "rev-parse",
15912                    "--path-format=absolute",
15913                    "--git-path",
15914                    "index.lock",
15915                ],
15916            )
15917            .await
15918            .unwrap(),
15919        );
15920        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15921        runner.state.schema = 14;
15922        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15923        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15924        assert!(runner.implement().await.is_err());
15925        assert!(runner.after_implement().is_err());
15926        assert!(runner.fold_losers().await.is_err());
15927        assert_eq!(
15928            std::fs::read_to_string(wt.join("recovered")).unwrap(),
15929            "failed seat edits"
15930        );
15931        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15932        runner.state = RunState::load(&runner.state.id).unwrap();
15933        std::fs::remove_file(index_lock).unwrap();
15934        // Recovery finds changes that the old uncontested result never judged.
15935        assert!(runner.implement().await.is_err());
15936        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15937        assert!(runner.state.candidates[0].empty);
15938        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15939        assert!(
15940            crate::run::read_artifact(&runner.state, "cand-A.patch")
15941                .unwrap()
15942                .contains("failed seat edits")
15943        );
15944        assert!(wt.exists());
15945    }
15946
15947    #[tokio::test]
15948    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
15949        let (_tmp, mut runner) = preservation_resume_fixture().await;
15950        let repo = runner.state.repo.clone();
15951        let common = git::git(
15952            &repo,
15953            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
15954        )
15955        .await
15956        .unwrap();
15957        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
15958        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
15959        assert!(runner.fold_losers().await.is_err());
15960        assert!(!runner.state.candidates[0].worktree.exists());
15961        assert!(!runner.state.candidates[0].folded);
15962        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
15963        runner.state = RunState::load(&runner.state.id).unwrap();
15964        std::fs::remove_file(branch_lock).unwrap();
15965        let judge = runner.state.worktree_root().join("judge-1");
15966        git::worktree_add_detached(&repo, &judge, "main")
15967            .await
15968            .unwrap();
15969        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
15970        runner.roles.judges = vec![spec("judge")];
15971        assert!(runner.fold_losers().await.is_err());
15972        runner.state = RunState::load(&runner.state.id).unwrap();
15973        assert!(
15974            runner.state.candidates[0].folded,
15975            "candidate progress saved before judge failure"
15976        );
15977        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15978        assert!(judge.join("uncommitted").exists());
15979        std::fs::remove_file(judge.join("uncommitted")).unwrap();
15980        runner.fold_losers().await.unwrap();
15981        assert!(!judge.exists());
15982        runner.fold_losers().await.unwrap();
15983    }
15984
15985    #[tokio::test]
15986    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15987        let (_tmp, mut runner) = preservation_resume_fixture().await;
15988        runner.state.candidates.retain(|c| c.label == 'B');
15989        runner.state.tally = None;
15990        runner.state.judge_skipped = false;
15991        runner.state.prep_complete = Some(true);
15992        runner.state.config.graph.advise = false;
15993        runner.state.config.graph.selected_implementer = Some("alpha".into());
15994        let wt = runner.state.candidates[0].worktree.clone();
15995        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15996        runner
15997            .state
15998            .candidate_preservation
15999            .insert('B', "awaiting persistence".into());
16000        let index_lock = git::git(
16001            &wt,
16002            &[
16003                "rev-parse",
16004                "--path-format=absolute",
16005                "--git-path",
16006                "index.lock",
16007            ],
16008        )
16009        .await
16010        .unwrap();
16011        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16012        runner.state.save().unwrap();
16013        assert!(runner.execute_graph().await.is_err());
16014        assert_eq!(runner.state.status, RunStatus::Blocked);
16015        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16016        std::fs::remove_file(index_lock).unwrap();
16017        // Park at the next graph boundary: this tests the real selected-seat
16018        // hold guard without dispatching reviewers or publishing anything.
16019        for _ in 0..2 {
16020            runner.state = RunState::load(&runner.state.id).unwrap();
16021            runner.pause.park();
16022            runner.execute_graph().await.unwrap();
16023            assert_eq!(runner.state.status, RunStatus::Implementing);
16024            assert!(runner.state.candidate_preservation.is_empty());
16025            assert_eq!(runner.state.candidates[0].commits, 2);
16026            assert!(runner.state.tally.is_none());
16027        }
16028        runner.implement().await.unwrap();
16029        assert_eq!(runner.state.status, RunStatus::Judging);
16030        // A genuine selected-implementer hold must still require release.
16031        runner.state.status = RunStatus::Blocked;
16032        runner.state.candidates[0].failed = Some("implementation failed".into());
16033        runner.state.save().unwrap();
16034        let error = runner.execute_graph().await.unwrap_err();
16035        assert!(
16036            error
16037                .to_string()
16038                .contains("selected implementation is held")
16039        );
16040    }
16041
16042    #[tokio::test]
16043    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16044        let (_tmp, mut runner) = preservation_resume_fixture().await;
16045        let repo = runner.state.repo.clone();
16046        let candidate = runner.state.candidates[0].clone();
16047        // Remove only the disposable fixture directory, leaving Git's linked
16048        // worktree registration and branch intact, as in a vanished checkout.
16049        let registered = git::worktree_holding(&repo, &candidate.branch)
16050            .await
16051            .unwrap()
16052            .unwrap();
16053        assert_eq!(
16054            registered.canonicalize().unwrap(),
16055            candidate.worktree.canonicalize().unwrap()
16056        );
16057        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16058        assert_eq!(
16059            git::worktree_holding(&repo, &candidate.branch)
16060                .await
16061                .unwrap(),
16062            Some(registered)
16063        );
16064        assert!(
16065            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16066            "the stale registration initially keeps the branch checked out"
16067        );
16068        runner.fold_losers().await.unwrap();
16069        assert!(runner.state.candidates[0].folded);
16070        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16071        assert_eq!(
16072            git::worktree_holding(&repo, &candidate.branch)
16073                .await
16074                .unwrap(),
16075            None
16076        );
16077        runner.state = RunState::load(&runner.state.id).unwrap();
16078        runner.fold_losers().await.unwrap();
16079    }
16080
16081    #[tokio::test]
16082    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16083        let (_tmp, mut runner) = preservation_resume_fixture().await;
16084        let repo = runner.state.repo.clone();
16085        let candidate = runner.state.candidates[0].clone();
16086        let winner = runner.state.candidates[1].clone();
16087        let registered = git::worktree_holding(&repo, &candidate.branch)
16088            .await
16089            .unwrap()
16090            .unwrap();
16091        assert_eq!(
16092            registered.canonicalize().unwrap(),
16093            candidate.worktree.canonicalize().unwrap()
16094        );
16095        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16096        assert_eq!(
16097            git::worktree_holding(&repo, &candidate.branch)
16098                .await
16099                .unwrap(),
16100            Some(registered)
16101        );
16102        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16103
16104        for _ in 0..2 {
16105            fold_run(&mut runner.state, false, &crate::run::home())
16106                .await
16107                .unwrap();
16108            runner.state = RunState::load(&runner.state.id).unwrap();
16109            assert!(runner.state.candidates[0].folded);
16110            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16111            assert_eq!(
16112                git::worktree_holding(&repo, &candidate.branch)
16113                    .await
16114                    .unwrap(),
16115                None
16116            );
16117            assert!(!runner.state.candidates[1].folded);
16118            assert!(winner.worktree.exists());
16119            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16120        }
16121    }
16122
16123    #[tokio::test]
16124    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16125        ask_test_home();
16126        let tmp = tempfile::tempdir().unwrap();
16127        let repo = tmp.path().join("repo");
16128        std::fs::create_dir(&repo).unwrap();
16129        init_repo(&repo);
16130        let mut runner = runner_at(RunStatus::Implementing);
16131        runner.state.repo = repo.clone();
16132        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16133        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16134        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16135            let branch = format!("candidate-{label}");
16136            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16137            git::worktree_add_branch(&repo, &wt, &branch, "main")
16138                .await
16139                .unwrap();
16140            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16141            git::git(&wt, &["add", "README.md"]).await.unwrap();
16142            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16143            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16144            runner.state.candidates.push(Candidate {
16145                index,
16146                label,
16147                agent: "alpha".to_owned(),
16148                branch,
16149                worktree: wt,
16150                summary: "completed implementation".to_owned(),
16151                stat: String::new(),
16152                files: 0,
16153                commits: 0,
16154                empty: false,
16155                failed: None,
16156                verified_noop: None,
16157                duration_ms: 42,
16158                folded: false,
16159            });
16160            runner
16161                .state
16162                .candidate_preservation
16163                .insert(label, "awaiting persistence".to_owned());
16164        }
16165        let wt = runner.state.candidates[0].worktree.clone();
16166        let index_lock = git::git(
16167            &wt,
16168            &[
16169                "rev-parse",
16170                "--path-format=absolute",
16171                "--git-path",
16172                "index.lock",
16173            ],
16174        )
16175        .await
16176        .unwrap();
16177        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16178        runner.state.save().unwrap();
16179        assert!(runner.persist_candidates().await.is_err());
16180        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16181        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16182        assert_eq!(
16183            runner.state.candidates[1].commits, 1,
16184            "other results are still collected"
16185        );
16186        assert!(
16187            !runner.state.candidates[0].empty,
16188            "failed rescue is not no change"
16189        );
16190        assert_eq!(
16191            runner.state.candidates[0].summary,
16192            "completed implementation"
16193        );
16194        assert!(runner.after_implement().is_err());
16195        let home = tmp.path().join("fold-home");
16196        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16197        assert!(wt.join("new-file").exists());
16198        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16199        assert!(recovery.join("staged.patch").exists());
16200        assert!(recovery.join("unstaged.patch").exists());
16201        assert_eq!(
16202            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16203            "untracked edits\n"
16204        );
16205        runner.state.begin_pass();
16206        runner.state.end_pass_parked(ParkCause::Upgrade {
16207            from: "1.0".into(),
16208            to: Some("1.1".into()),
16209        });
16210        runner.state.save().unwrap();
16211        runner.state = RunState::load(&runner.state.id).unwrap();
16212        assert!(matches!(
16213            runner.state.passes.last().unwrap().end,
16214            Some(crate::run::PassEnd::Parked {
16215                cause: ParkCause::Upgrade { .. }
16216            })
16217        ));
16218        std::fs::remove_file(index_lock).unwrap();
16219        // No implementer is configured: a persistence retry cannot dispatch a seat.
16220        runner.implement().await.unwrap();
16221        assert!(runner.state.candidate_preservation.is_empty());
16222        assert!(
16223            runner
16224                .state
16225                .candidates
16226                .iter()
16227                .all(|c| c.commits == 1 && !c.empty)
16228        );
16229        assert!(git::is_clean(&wt).await.unwrap());
16230        assert!(
16231            crate::run::read_artifact(&runner.state, "cand-A.patch")
16232                .unwrap()
16233                .contains("later tracked edits")
16234        );
16235    }
16236
16237    #[tokio::test]
16238    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16239        ask_test_home();
16240        let tmp = tempfile::tempdir().unwrap();
16241        let repo = tmp.path().join("repo");
16242        std::fs::create_dir(&repo).unwrap();
16243        init_repo(&repo);
16244        let wt = tmp.path().join("candidate");
16245        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16246            .await
16247            .unwrap();
16248        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16249        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16250        assert!(wt.join("uncommitted").exists());
16251        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16252        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16253        let mut runner = runner_at(RunStatus::Implementing);
16254        runner.state.repo = repo.clone();
16255        runner.state.base_commit = "missing-base".to_owned();
16256        runner.state.candidates.push(Candidate {
16257            index: 0,
16258            label: 'A',
16259            agent: "alpha".into(),
16260            branch: "candidate".into(),
16261            worktree: wt.clone(),
16262            summary: "completed".into(),
16263            stat: String::new(),
16264            files: 0,
16265            commits: 0,
16266            empty: false,
16267            failed: None,
16268            verified_noop: None,
16269            duration_ms: 1,
16270            folded: false,
16271        });
16272        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16273        assert!(
16274            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16275                .await
16276                .is_err()
16277        );
16278        assert!(!runner.state.candidates[0].folded);
16279        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16280        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16281        runner
16282            .state
16283            .candidate_preservation
16284            .insert('A', "pending".into());
16285        assert!(runner.persist_candidates().await.is_err());
16286        assert!(!runner.state.candidates[0].empty);
16287        assert!(!runner.state.candidates[0].folded);
16288        assert!(wt.exists());
16289        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16290        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16291        std::fs::create_dir_all(&patch_path).unwrap();
16292        assert!(
16293            runner.persist_candidates().await.is_err(),
16294            "artifact failure stays pending"
16295        );
16296        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16297        assert!(!runner.state.candidates[0].empty);
16298        std::fs::remove_dir(patch_path).unwrap();
16299        runner.persist_candidates().await.unwrap();
16300        assert!(
16301            runner.state.candidates[0].empty,
16302            "only successful inspection confirms no change"
16303        );
16304    }
16305
16306    fn init_repo(dir: &Path) {
16307        let run = |args: &[&str]| {
16308            let out = std::process::Command::new("git")
16309                .args(args)
16310                .current_dir(dir)
16311                .quiet()
16312                .output()
16313                .expect("spawn git");
16314            assert!(
16315                out.status.success(),
16316                "git {args:?} failed: {}",
16317                String::from_utf8_lossy(&out.stderr)
16318            );
16319        };
16320        run(&["init", "-b", "main"]);
16321        run(&["config", "user.name", "magi test"]);
16322        run(&["config", "user.email", "magi@example.com"]);
16323        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16324        run(&["add", "-A"]);
16325        run(&["commit", "-m", "init"]);
16326    }
16327
16328    // `settle_questions` is what closes the ghost the phone showed: a run's
16329    // seat asked something, the run then ended, and nothing was left to
16330    // abandon the question it left `open`. `HOME` is a process-wide
16331    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16332    // first time it runs in the binary — every test below still reaches the
16333    // same directory whichever call won, and each gets its own run id from
16334    // `RunState::new`, so they never collide there.
16335    fn ask_test_home() {
16336        crate::run::pin_test_home();
16337    }
16338
16339    #[test]
16340    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16341        ask_test_home();
16342        let mut runner = runner_at(RunStatus::Prep);
16343        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16344        // A blocked temporary output path makes the terminal save fail while
16345        // preserving the already published reservation.
16346        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16347        let error = guard
16348            .finish(
16349                &mut runner.state,
16350                Err(
16351                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16352                        .into(),
16353                ),
16354            )
16355            .unwrap_err();
16356        assert_eq!(
16357            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16358            std::io::ErrorKind::PermissionDenied
16359        );
16360        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16361        let message = format!("{error:#}");
16362        assert!(message.contains("original refusal"));
16363        assert!(!message.contains("could not save terminal"));
16364        assert_eq!(
16365            error.chain().count(),
16366            2,
16367            "only startup attribution wraps the original error"
16368        );
16369        let saved = RunState::load(&runner.state.id).unwrap();
16370        assert_eq!(saved.status, RunStatus::Blocked);
16371        assert!(!saved.startup.unwrap().complete);
16372    }
16373
16374    #[test]
16375    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16376        ask_test_home();
16377        let mut runner = runner_at(RunStatus::Prep);
16378        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16379        let id = runner.state.id.clone();
16380        drop(guard);
16381        let saved = RunState::load(&id).unwrap();
16382        assert_eq!(saved.status, RunStatus::Blocked);
16383        assert!(!saved.startup.as_ref().unwrap().complete);
16384        assert!(
16385            saved
16386                .startup
16387                .unwrap()
16388                .failure
16389                .unwrap()
16390                .contains("interrupted")
16391        );
16392        assert!(Runner::resume(&id).is_err());
16393    }
16394
16395    #[test]
16396    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16397        ask_test_home();
16398        let mut runner = runner_at(RunStatus::Prep);
16399        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16400        let error = guard
16401            .finish(
16402                &mut runner.state,
16403                Err(
16404                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16405                        .into(),
16406                ),
16407            )
16408            .unwrap_err();
16409        assert_eq!(
16410            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16411            std::io::ErrorKind::PermissionDenied
16412        );
16413        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16414        let mut task = crate::queue::Task::new(
16415            "review".into(),
16416            "criteria".into(),
16417            runner.state.repo.clone(),
16418            crate::queue::Source::Human,
16419        );
16420        task.record_startup_failure(failure.clone());
16421        task.record_startup_failure(failure.clone());
16422        assert_eq!(task.startup_failures.len(), 1);
16423        assert!(task.runs.is_empty());
16424        assert_eq!(task.attempts, 0);
16425    }
16426
16427    #[cfg(unix)]
16428    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16429        ask_test_home();
16430        let tmp = tempfile::tempdir().unwrap();
16431        let repo = tmp.path().join("repo");
16432        std::fs::create_dir(&repo).unwrap();
16433        init_repo(&repo);
16434        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16435        let origin = tmp.path().join("origin.git");
16436        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16437            .await
16438            .unwrap();
16439        git::git(
16440            &repo,
16441            &["remote", "add", "origin", origin.to_str().unwrap()],
16442        )
16443        .await
16444        .unwrap();
16445        git::git(&repo, &["checkout", "-b", "fix/retained"])
16446            .await
16447            .unwrap();
16448        std::fs::write(repo.join("retained"), "retained change").unwrap();
16449        git::git(&repo, &["add", "."]).await.unwrap();
16450        git::git(&repo, &["commit", "-m", "retained implementation"])
16451            .await
16452            .unwrap();
16453        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16454        git::git(&repo, &["checkout", "main"]).await.unwrap();
16455        std::fs::write(repo.join("base-change"), "new base").unwrap();
16456        git::git(&repo, &["add", "."]).await.unwrap();
16457        git::git(&repo, &["commit", "-m", "base moved"])
16458            .await
16459            .unwrap();
16460        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16461        git::git(&repo, &["checkout", "fix/retained"])
16462            .await
16463            .unwrap();
16464        let mut runner = runner_at(RunStatus::Voting);
16465        runner.state.repo = repo.clone();
16466        runner.state.base_commit = base;
16467        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16468        runner.state.config.disk.min_free_bytes = 0;
16469        runner.state.config.graph.review_rounds = 1;
16470        runner.state.config.verify.e2e = vec!["true".into()];
16471        runner.state.config.verify.gate = vec!["true".into()];
16472        runner.state.candidates = vec![Candidate {
16473            index: 0,
16474            label: 'A',
16475            agent: "fixture".into(),
16476            branch: "fix/retained".into(),
16477            worktree: repo.clone(),
16478            summary: String::new(),
16479            stat: String::new(),
16480            files: 1,
16481            commits: 1,
16482            empty: false,
16483            failed: None,
16484            verified_noop: None,
16485            duration_ms: 0,
16486            folded: false,
16487        }];
16488        runner.state.tally = Some(Tally {
16489            first_choice: BTreeMap::new(),
16490            borda: BTreeMap::new(),
16491            winner: 'A',
16492            rankings: 0,
16493            unanimous_initial: true,
16494            deliberated: false,
16495            changed_votes: 0,
16496            unanimous_final: true,
16497            tie_break: None,
16498            judges: 0,
16499            present: 0,
16500            quorum: 0,
16501            met_quorum: true,
16502            uncontested: Some("review-only".into()),
16503        });
16504        let mut task = crate::queue::Task::new(
16505            "fixture".into(),
16506            "fixture".into(),
16507            repo,
16508            crate::queue::Source::Human,
16509        );
16510        task.link_run(&runner.state.id);
16511        crate::queue::Queue::open().put(&mut task).unwrap();
16512        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16513        runner.state.base_sync = Some(BaseSync {
16514            tip: String::new(),
16515            behind: 1,
16516            attempts: BASE_SYNC_ROUNDS,
16517            conflict: None,
16518            already_in: None,
16519        });
16520        runner.sync_to_base().await.unwrap();
16521        assert_eq!(runner.state.status, RunStatus::Blocked);
16522        (tmp, runner, head, task.id)
16523    }
16524
16525    #[cfg(unix)]
16526    #[tokio::test]
16527    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16528        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16529        if std::env::var_os(CHILD).is_none() {
16530            let home = tempfile::tempdir().unwrap();
16531            let output = std::process::Command::new(std::env::current_exe().unwrap())
16532                .args([
16533                    "--exact",
16534                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16535                    "--nocapture",
16536                ])
16537                .env(CHILD, "1")
16538                .env("MAGI_HOME", home.path())
16539                .env("MAGI_WORKER_HOME", home.path())
16540                .env("MAGI_WORKER_ID", "abcdef")
16541                .output()
16542                .unwrap();
16543            assert!(
16544                output.status.success()
16545                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16546                "{}\n{}",
16547                String::from_utf8_lossy(&output.stdout),
16548                String::from_utf8_lossy(&output.stderr)
16549            );
16550            return;
16551        }
16552        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16553        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16554        let home = crate::run::home();
16555        let queue = crate::queue::Queue::open();
16556        let mut owner = queue.get(&task).unwrap();
16557        owner.release_operator("test-release", None);
16558        owner.start(runner.state.id.clone());
16559        queue.put(&mut owner).unwrap();
16560        runner.state.driver_exited = true;
16561        runner.state.task_instruction = Some(owner.instruction.clone());
16562        runner.state.save().unwrap();
16563        let _claim = queue.claim(&task).unwrap();
16564        let reserve = |id: &str| {
16565            let directory = home.join("workers/jobs").join(id);
16566            std::fs::create_dir_all(&directory).unwrap();
16567            let job = crate::worker::Job {
16568                id: id.into(),
16569                sequence: 1,
16570                kind: "task".into(),
16571                key: format!("task:{task}"),
16572                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16573                generation: "fixture-generation".into(),
16574                version: None,
16575                status: crate::worker::Status::Running,
16576                result: None,
16577                pid: Some(std::process::id()),
16578                created_at: 0,
16579                finished_at: None,
16580            };
16581            std::fs::write(
16582                directory.join("job.json"),
16583                serde_json::to_vec(&job).unwrap(),
16584            )
16585            .unwrap();
16586            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16587        };
16588        let _own = reserve("abcdef");
16589        let other = reserve("fedcba");
16590        assert!(
16591            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16592                .await
16593                .is_err()
16594        );
16595        assert!(
16596            RunState::load(&runner.state.id)
16597                .unwrap()
16598                .base_sync_recovery
16599                .authorizations
16600                .is_empty()
16601        );
16602        drop(other);
16603        assert!(crate::daemon::is_working_on(
16604            &home,
16605            &runner.state.id,
16606            Timestamp::now()
16607        ));
16608        assert!(
16609            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16610        );
16611        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16612            .await
16613            .unwrap();
16614        assert!(
16615            RunState::load(&runner.state.id)
16616                .unwrap()
16617                .base_recovery_pending()
16618        );
16619    }
16620
16621    #[cfg(unix)]
16622    #[tokio::test]
16623    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16624        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16625        let home = crate::run::home();
16626        runner.state.driver_pid = Some(std::process::id());
16627        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16628        runner.state.driver_exited = false;
16629        runner.state.begin_pass();
16630        runner.state.save().unwrap();
16631        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16632        driver.record(&runner.state).unwrap();
16633        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16634        // Preserve the durable claim as if the process died before Drop saved.
16635        drop(driver);
16636        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16637        assert_eq!(
16638            runner.state.liveness_under(&home, false),
16639            crate::run::Liveness::Dead
16640        );
16641        let queue = crate::queue::Queue::open();
16642        let _claim = queue.claim(&task).unwrap();
16643        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16644            .await
16645            .unwrap();
16646        let disk = RunState::load(&runner.state.id).unwrap();
16647        assert!(!disk.driver_exited);
16648        assert!(disk.base_recovery_pending());
16649    }
16650
16651    #[cfg(unix)]
16652    #[tokio::test]
16653    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16654        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16655        if std::env::var_os(CHILD).is_none() {
16656            use std::os::unix::fs::PermissionsExt;
16657            let tmp = tempfile::tempdir().unwrap();
16658            let gh = tmp.path().join("gh");
16659            std::fs::write(&gh, r#"#!/bin/sh
16660case "$1 $2" in
16661  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16662  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16663  *) exit 2 ;;
16664esac
16665"#).unwrap();
16666            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16667            let mut paths = vec![tmp.path().to_path_buf()];
16668            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16669            let output = std::process::Command::new(std::env::current_exe().unwrap())
16670                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16671                .env(CHILD, "1")
16672                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16673                .env("PATH", std::env::join_paths(paths).unwrap())
16674                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16675                .output().unwrap();
16676            assert!(
16677                output.status.success()
16678                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16679                "{}\n{}",
16680                String::from_utf8_lossy(&output.stdout),
16681                String::from_utf8_lossy(&output.stderr)
16682            );
16683            return;
16684        }
16685        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16686        let repo = runner.state.repo.clone();
16687        git::git(&repo, &["checkout", "main"]).await.unwrap();
16688        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16689        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16690        git::git(&repo, &["checkout", "fix/retained"])
16691            .await
16692            .unwrap();
16693        runner.state.pr = Some(crate::run::PrRecord {
16694            url: "https://example.test/project/pull/1".into(),
16695            number: 1,
16696            state: "open".into(),
16697            checks: "unknown".into(),
16698            round: 0,
16699            rounds: 1,
16700            red_at_merge: Vec::new(),
16701        });
16702        runner.sync_to_base().await.unwrap();
16703        assert_eq!(runner.state.status, RunStatus::Blocked);
16704        assert_eq!(
16705            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16706            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16707        );
16708        assert!(runner.state.base_sync_closure_pending());
16709        assert!(crate::handover::can_resume(&runner.state));
16710        let queue = crate::queue::Queue::open();
16711        assert!(!crate::base_sync_recovery::held_task(
16712            &queue,
16713            &queue.get(&task).unwrap()
16714        ));
16715        assert!(runner.state.authorize_base_sync(&task).is_err());
16716        // Legacy closure stops have no typed stop but must retain the checkpoint.
16717        runner.state.base_sync_recovery.stop = None;
16718        runner.state.status = RunStatus::Voting;
16719        runner.state.gate_ran = true;
16720        runner.state.config.graph.review_rounds = 0;
16721        runner.state.save().unwrap();
16722        assert!(crate::handover::can_resume(&runner.state));
16723        runner.state = RunState::load(&runner.state.id).unwrap();
16724        runner.execute_graph().await.unwrap();
16725        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16726        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16727        assert_eq!(
16728            runner.state.base_sync.as_ref().unwrap().attempts,
16729            BASE_SYNC_ROUNDS
16730        );
16731        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16732        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16733        assert!(runner.state.reviews.is_empty());
16734        assert!(runner.state.gate.is_empty());
16735    }
16736
16737    #[cfg(unix)]
16738    #[tokio::test]
16739    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16740        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16741        runner.state.driver_exited = true;
16742        runner.state.save().unwrap();
16743        let home = crate::run::home();
16744        let queue = crate::queue::Queue::open();
16745        let _claim = queue.claim(&task).unwrap();
16746        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16747        assert!(
16748            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16749                .await
16750                .is_err()
16751        );
16752        drop(lease);
16753        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16754        assert!(
16755            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16756                .await
16757                .is_err()
16758        );
16759        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16760        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16761            .await
16762            .unwrap();
16763        let disk = RunState::load(&runner.state.id).unwrap();
16764        assert!(disk.base_recovery_pending());
16765        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16766        assert!(
16767            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16768                .await
16769                .is_err()
16770        );
16771        assert_eq!(
16772            RunState::load(&runner.state.id)
16773                .unwrap()
16774                .base_sync_recovery
16775                .authorizations
16776                .len(),
16777            1
16778        );
16779        let mut owner = queue.get(&task).unwrap();
16780        owner.runs.push("20260101-000000-ffff".into());
16781        queue.put(&mut owner).unwrap();
16782        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16783            .await
16784            .unwrap_err();
16785        assert!(error.to_string().contains("run task ownership changed"));
16786    }
16787
16788    #[cfg(unix)]
16789    #[tokio::test]
16790    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16791        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16792        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16793        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16794        legacy["status"] = serde_json::json!("voting");
16795        runner.state = serde_json::from_value(legacy).unwrap();
16796        runner.state.save().unwrap();
16797        for _ in 0..3 {
16798            runner.execute_graph().await.unwrap();
16799            assert_eq!(runner.state.status, RunStatus::Blocked);
16800            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16801            assert!(!crate::handover::can_resume(&runner.state));
16802            assert_eq!(
16803                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16804                head
16805            );
16806            assert!(runner.state.reviews.is_empty());
16807            assert!(!runner.state.gate_ran);
16808        }
16809        let notice = crate::notices::run_ended(&runner.state).unwrap();
16810        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16811        runner.state.begin_pass();
16812        runner.state.status = RunStatus::Voting;
16813        runner.deliberate().await.unwrap();
16814        runner.vote().await.unwrap();
16815        runner.finish_after_tally().await.unwrap();
16816        assert_eq!(runner.state.status, RunStatus::Blocked);
16817        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16818        assert_eq!(
16819            crate::notices::run_ended(&runner.state).unwrap().since,
16820            notice.since
16821        );
16822    }
16823
16824    #[cfg(unix)]
16825    #[tokio::test]
16826    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16827        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16828        for detail in [
16829            "fix/retained conflicts with origin/main and did not rebase: conflict",
16830            "rebased fix/retained locally but origin refused the push",
16831            "unknown failure",
16832        ] {
16833            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16834            runner.state.record_base_sync_stop(&head);
16835            assert!(runner.state.authorize_base_sync(&task).is_err());
16836            runner.state.status = RunStatus::Voting;
16837            runner.execute_graph().await.unwrap();
16838            assert_eq!(runner.state.status, RunStatus::Blocked);
16839            assert!(!crate::handover::can_resume(&runner.state));
16840            assert_eq!(
16841                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16842                head
16843            );
16844        }
16845    }
16846
16847    #[cfg(unix)]
16848    #[tokio::test]
16849    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16850        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16851        runner.state.authorize_base_sync(&task).unwrap();
16852        assert!(runner.state.consume_base_recovery(&head));
16853        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
16854        runner.state.reserve_base_operation(&head, &tip);
16855        runner.state.save().unwrap();
16856        runner.state = RunState::load(&runner.state.id).unwrap();
16857        assert!(!runner.state.consume_base_recovery(&head));
16858        runner.sync_to_base().await.unwrap();
16859        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16860        assert_eq!(runner.state.base_episode_used(), 1);
16861        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16862        assert!(
16863            runner
16864                .state
16865                .base_sync_recovery
16866                .operations
16867                .last()
16868                .unwrap()
16869                .finished
16870        );
16871        assert!(!runner.state.base_sync_blocked());
16872        runner.sync_to_base().await.unwrap();
16873        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16874    }
16875
16876    #[cfg(unix)]
16877    #[tokio::test]
16878    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
16879        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16880        // Both historical diagnostic spellings retain an exhaustion identity.
16881        let original = runner
16882            .state
16883            .base_sync
16884            .as_ref()
16885            .unwrap()
16886            .conflict
16887            .clone()
16888            .unwrap();
16889        runner.state.base_sync.as_mut().unwrap().conflict = Some(
16890            original
16891                .replace("synchronization(s)", "rebase(s)")
16892                .replace("synchronizing", "rebasing"),
16893        );
16894        runner.state.base_sync_recovery = Default::default();
16895        runner.state.record_base_sync_stop(&head);
16896        assert_eq!(
16897            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16898            crate::base_sync_recovery::StopKind::Exhausted
16899        );
16900        runner.state.authorize_base_sync(&task).unwrap();
16901        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16902        let auth = legacy["base_sync_recovery"]["authorizations"][0]
16903            .as_object_mut()
16904            .unwrap();
16905        auth.remove("interval");
16906        auth.remove("scope");
16907        runner.state = serde_json::from_value(legacy).unwrap();
16908        assert!(runner.state.consume_base_recovery(&head));
16909        assert_eq!(runner.state.base_episode_used(), 3);
16910        runner.sync_to_base().await.unwrap();
16911        assert_eq!(runner.state.base_episode_used(), 4);
16912        runner.state.resource_block = Some(crate::run::ResourceBlock {
16913            node: "gate".into(),
16914            kind: "cache".into(),
16915            detail: "legacy wait".into(),
16916            at: Timestamp::now(),
16917            cause: crate::run::ResourceBlockCause::Unrecorded,
16918        });
16919        runner.state.save().unwrap();
16920        assert!(
16921            runner
16922                .state
16923                .base_sync_recovery
16924                .episodes
16925                .last()
16926                .unwrap()
16927                .boundary
16928                .is_none()
16929        );
16930        runner.state.resource_block = None;
16931        advance_base_fixture(&runner).await;
16932        runner.sync_to_base().await.unwrap();
16933        assert!(runner.state.base_sync_blocked());
16934        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16935    }
16936
16937    #[cfg(unix)]
16938    async fn advance_base_fixture(runner: &Runner) {
16939        let repo = &runner.state.repo;
16940        for args in [
16941            vec!["checkout", "main"],
16942            vec!["commit", "--allow-empty", "-m", "base advanced"],
16943            vec!["push", "origin", "main"],
16944            vec!["checkout", "fix/retained"],
16945        ] {
16946            git::git(repo, &args).await.unwrap();
16947        }
16948    }
16949
16950    #[cfg(unix)]
16951    fn fresh_base_interval_fixture(runner: &mut Runner) {
16952        runner.state.base_sync.as_mut().unwrap().attempts = 0;
16953        runner.state.base_sync.as_mut().unwrap().conflict = None;
16954        runner.state.base_sync_recovery = Default::default();
16955        runner.state.status = RunStatus::Voting;
16956    }
16957
16958    #[cfg(unix)]
16959    #[tokio::test]
16960    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
16961        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
16962        fresh_base_interval_fixture(&mut runner);
16963        runner.state.begin_pass();
16964        runner.sync_to_base().await.unwrap();
16965        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16966        runner.state.resource_block = Some(crate::run::ResourceBlock {
16967            node: "gate".into(),
16968            kind: "build cache".into(),
16969            detail: "busy".into(),
16970            at: Timestamp::now(),
16971            cause: crate::run::ResourceBlockCause::Timeout {
16972                waited_secs: 1,
16973                limit_secs: 1,
16974            },
16975        });
16976        runner.state.save().unwrap();
16977        let queue = crate::queue::Queue::open();
16978        let mut owner = queue.get(&task).unwrap();
16979        runner.state.task_instruction = Some(owner.instruction.clone());
16980        owner.release_operator("fixture-owner", None);
16981        queue.put(&mut owner).unwrap();
16982        let approval = owner.resumption.clone().unwrap();
16983        runner.state.resource_block = None;
16984        runner.state.save().unwrap();
16985        runner.state = RunState::load(&runner.state.id).unwrap();
16986        advance_base_fixture(&runner).await;
16987        runner.sync_to_base().await.unwrap();
16988        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
16989        assert_eq!(runner.state.base_episode_used(), 1);
16990        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16991        assert!(
16992            runner.state.base_sync_recovery.episodes[0]
16993                .boundary
16994                .as_ref()
16995                .unwrap()
16996                .consumed
16997        );
16998        assert!(
16999            !runner
17000                .state
17001                .renew_base_episode(&task, &approval, &head, "base")
17002        );
17003        for _ in 0..3 {
17004            owner.release_operator("fixture-owner", None);
17005            queue.put(&mut owner).unwrap();
17006            runner.state = RunState::load(&runner.state.id).unwrap();
17007            runner.state.begin_pass();
17008            runner.sync_to_base().await.unwrap();
17009            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17010            assert_eq!(runner.state.base_episode_used(), 1);
17011        }
17012        runner.state.resource_block = Some(crate::run::ResourceBlock {
17013            node: "gate".into(),
17014            kind: "disk pressure".into(),
17015            detail: "free space required".into(),
17016            at: Timestamp::now(),
17017            cause: crate::run::ResourceBlockCause::Other,
17018        });
17019        runner.state.end_pass_parked(ParkCause::Interrupt {
17020            reason: "disk pressure".into(),
17021        });
17022        runner.state.save().unwrap();
17023        owner.release_operator("fixture-owner-after-disk", None);
17024        queue.put(&mut owner).unwrap();
17025        runner.state.resource_block = None;
17026        advance_base_fixture(&runner).await;
17027        runner.sync_to_base().await.unwrap();
17028        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17029        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17030        assert_eq!(runner.state.base_episode_used(), 1);
17031        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17032    }
17033
17034    #[cfg(unix)]
17035    #[tokio::test]
17036    async fn base_recovery_continuous_advancement_stops_at_four() {
17037        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17038        fresh_base_interval_fixture(&mut runner);
17039        for spent in 1..=BASE_SYNC_ROUNDS {
17040            runner.sync_to_base().await.unwrap();
17041            assert!(!runner.state.base_sync_blocked());
17042            assert_eq!(runner.state.base_episode_used(), spent);
17043            advance_base_fixture(&runner).await;
17044        }
17045        runner.sync_to_base().await.unwrap();
17046        assert_eq!(
17047            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17048            crate::base_sync_recovery::StopKind::Exhausted
17049        );
17050        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17051        assert!(stop.unverified_sync);
17052        let action = runner.state.base_sync_next_action();
17053        assert!(action.contains("never reached its gate"));
17054        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17055        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17056        runner.state = RunState::load(&runner.state.id).unwrap();
17057        runner.sync_to_base().await.unwrap();
17058        assert_eq!(runner.state.base_episode_used(), 4);
17059        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17060    }
17061
17062    #[cfg(unix)]
17063    #[tokio::test]
17064    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17065        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17066        runner.state.ensure_base_episode();
17067        // Verification progress alone is not evidence of a successful sync.
17068        runner.state.record_base_progress(&head);
17069        runner.state.base_sync_recovery.stop = None;
17070        runner.state.record_base_sync_stop(&head);
17071        assert!(
17072            !runner
17073                .state
17074                .base_sync_recovery
17075                .stop
17076                .as_ref()
17077                .unwrap()
17078                .unverified_sync
17079        );
17080
17081        runner.state.finish_base_operation(Some(&head));
17082        // A gate for another head cannot verify this synchronization.
17083        runner.state.gate_ran = true;
17084        runner.state.gate_head = Some("older-head".into());
17085        runner.state.base_sync_recovery.stop = None;
17086        runner.state.record_base_sync_stop(&head);
17087        runner.state.save().unwrap();
17088        runner.state = RunState::load(&runner.state.id).unwrap();
17089        assert!(
17090            runner
17091                .state
17092                .base_sync_recovery
17093                .stop
17094                .as_ref()
17095                .unwrap()
17096                .unverified_sync
17097        );
17098
17099        runner.state.gate_head = Some(head.clone());
17100        runner.state.base_sync_recovery.stop = None;
17101        runner.state.record_base_sync_stop(&head);
17102        assert!(
17103            !runner
17104                .state
17105                .base_sync_recovery
17106                .stop
17107                .as_ref()
17108                .unwrap()
17109                .unverified_sync
17110        );
17111        assert!(
17112            !runner
17113                .state
17114                .base_sync_next_action()
17115                .contains("never reached its gate")
17116        );
17117
17118        // Historical records cannot establish the missing synchronization evidence.
17119        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17120        legacy["base_sync_recovery"]["stop"]
17121            .as_object_mut()
17122            .unwrap()
17123            .remove("unverified_sync");
17124        legacy["base_sync_recovery"]["episodes"][0]
17125            .as_object_mut()
17126            .unwrap()
17127            .remove("last_sync_head");
17128        runner.state = serde_json::from_value(legacy).unwrap();
17129        assert!(
17130            !runner
17131                .state
17132                .base_sync_recovery
17133                .stop
17134                .as_ref()
17135                .unwrap()
17136                .unverified_sync
17137        );
17138        assert!(
17139            runner.state.base_sync_recovery.episodes[0]
17140                .last_sync_head
17141                .is_none()
17142        );
17143    }
17144
17145    #[cfg(unix)]
17146    #[tokio::test]
17147    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17148        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17149        runner.state.authorize_base_sync(&task).unwrap();
17150        assert!(
17151            runner
17152                .state
17153                .base_sync_next_action()
17154                .contains("Approval recorded")
17155        );
17156        assert!(
17157            runner
17158                .state
17159                .base_sync_next_action()
17160                .contains("4/4 in episode 0, 4 lifetime")
17161        );
17162        assert!(runner.state.consume_base_recovery(&head));
17163        assert!(
17164            runner
17165                .state
17166                .base_sync_next_action()
17167                .contains("0/4 in episode 1, 4 lifetime")
17168        );
17169    }
17170
17171    #[cfg(unix)]
17172    #[tokio::test]
17173    async fn base_recovery_changed_head_invalidates_without_spending() {
17174        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17175        runner.state.authorize_base_sync(&task).unwrap();
17176        git::git(
17177            &runner.state.repo,
17178            &["commit", "--allow-empty", "-m", "external head change"],
17179        )
17180        .await
17181        .unwrap();
17182        runner.sync_to_base().await.unwrap();
17183        let a = runner
17184            .state
17185            .base_sync_recovery
17186            .authorizations
17187            .last()
17188            .unwrap();
17189        assert!(a.consumed);
17190        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17191        runner.state = RunState::load(&runner.state.id).unwrap();
17192        assert!(
17193            runner
17194                .state
17195                .base_sync_next_action()
17196                .contains("Approval invalidated")
17197        );
17198        assert!(!runner.state.base_recovery_pending());
17199        assert!(runner.state.base_sync_blocked());
17200        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17201    }
17202
17203    #[cfg(unix)]
17204    #[tokio::test]
17205    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17206        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17207        runner.state.driver_exited = true;
17208        runner.state.save().unwrap();
17209        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17210            .await
17211            .unwrap();
17212        runner.state = RunState::load(&runner.state.id).unwrap();
17213        advance_base_fixture(&runner).await;
17214        runner.sync_to_base().await.unwrap();
17215        let a = runner
17216            .state
17217            .base_sync_recovery
17218            .authorizations
17219            .last()
17220            .unwrap();
17221        assert!(
17222            a.consumed
17223                && a.invalidated
17224                    .as_deref()
17225                    .unwrap()
17226                    .contains("fetched base changed")
17227        );
17228        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17229        assert_eq!(
17230            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17231            head
17232        );
17233        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17234            .await
17235            .unwrap();
17236        runner.state = RunState::load(&runner.state.id).unwrap();
17237        runner.sync_to_base().await.unwrap();
17238        assert!(!runner.state.base_sync_blocked());
17239        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17240    }
17241
17242    #[cfg(unix)]
17243    #[tokio::test]
17244    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17245        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17246        runner.state.authorize_base_sync(&task).unwrap();
17247        assert!(runner.state.consume_base_recovery(&head));
17248        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17249        runner.state.reserve_base_operation(&head, &tip);
17250        runner.state.save().unwrap();
17251        let scratch = runner.state.dir().join("base-sync");
17252        assert_eq!(
17253            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17254                .await
17255                .unwrap(),
17256            crate::rebase::Rebased::Applied
17257        );
17258        runner.state = RunState::load(&runner.state.id).unwrap();
17259        runner.sync_to_base().await.unwrap();
17260        assert!(
17261            runner
17262                .state
17263                .base_sync_recovery
17264                .operations
17265                .last()
17266                .unwrap()
17267                .finished
17268        );
17269        assert_eq!(runner.state.base_episode_used(), 1);
17270        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17271        assert!(!runner.state.base_sync_blocked());
17272    }
17273
17274    #[cfg(unix)]
17275    #[tokio::test]
17276    async fn base_recovery_crash_preserves_untracked_files() {
17277        for ignored in [false, true] {
17278            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17279            runner.state.authorize_base_sync(&task).unwrap();
17280            assert!(runner.state.consume_base_recovery(&head));
17281            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17282            runner.state.reserve_base_operation(&head, &tip);
17283            runner.state.save().unwrap();
17284            let scratch = runner.state.dir().join("base-sync");
17285            assert_eq!(
17286                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17287                    .await
17288                    .unwrap(),
17289                crate::rebase::Rebased::Applied
17290            );
17291            let repo = &runner.state.repo;
17292            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17293            assert_ne!(result, head);
17294            if ignored {
17295                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17296                    .await
17297                    .unwrap();
17298                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17299            }
17300            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17301            assert!(
17302                git::git_raw(repo, &["diff", "--quiet", &head])
17303                    .await
17304                    .unwrap()
17305                    .ok()
17306            );
17307            // Resume after the branch moved, before its checkout was reset.
17308            runner.state = RunState::load(&runner.state.id).unwrap();
17309            runner.sync_to_base().await.unwrap();
17310            assert!(runner.state.base_sync_blocked());
17311            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17312            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17313            assert_eq!(
17314                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17315                result
17316            );
17317            assert_eq!(
17318                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17319                "retained work\n"
17320            );
17321            assert!(
17322                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17323                    .await
17324                    .unwrap()
17325                    .ok()
17326            );
17327        }
17328    }
17329
17330    #[cfg(unix)]
17331    #[tokio::test]
17332    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17333        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17334        runner.state.authorize_base_sync(&task).unwrap();
17335        assert!(runner.state.consume_base_recovery(&head));
17336        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17337        runner.state.reserve_base_operation(&head, &tip);
17338        runner.state.save().unwrap();
17339        let scratch = runner.state.dir().join("base-sync");
17340        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17341            .await
17342            .unwrap();
17343        git::sync_to_head(&runner.state.repo).await.unwrap();
17344        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17345        git::git(&runner.state.repo, &["add", "foreign"])
17346            .await
17347            .unwrap();
17348        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17349            .await
17350            .unwrap();
17351        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17352        runner.state = RunState::load(&runner.state.id).unwrap();
17353        runner.sync_to_base().await.unwrap();
17354        assert!(runner.state.base_sync_blocked());
17355        assert!(
17356            !runner
17357                .state
17358                .base_sync_recovery
17359                .operations
17360                .last()
17361                .unwrap()
17362                .finished
17363        );
17364        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17365        assert_eq!(
17366            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17367            foreign_head
17368        );
17369    }
17370
17371    #[cfg(unix)]
17372    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17373    {
17374        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17375        fresh_base_interval_fixture(&mut runner);
17376        let repo = &runner.state.repo;
17377        git::git(repo, &["checkout", "main"]).await.unwrap();
17378        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17379        git::git(repo, &["add", "retained"]).await.unwrap();
17380        git::git(repo, &["commit", "-m", "upstream retained change"])
17381            .await
17382            .unwrap();
17383        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17384        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17385        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17386        let scratch = runner.state.dir().join("base-sync");
17387        assert!(matches!(
17388            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17389                .await
17390                .unwrap(),
17391            git::RebaseStart::Conflicted(_)
17392        ));
17393        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17394            agent: "fixture".into(),
17395            paths: vec!["retained".into()],
17396            from: Some(head.clone()),
17397            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17398            index_outside: None,
17399            finished: false,
17400            error: None,
17401            launcher: None,
17402            infra: None,
17403            outage: None,
17404        });
17405        (tmp, runner, head, tip, scratch)
17406    }
17407
17408    #[cfg(unix)]
17409    #[tokio::test]
17410    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17411        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17412        // A resource stop from before reservations existed, with resolved
17413        // contents left for the launcher's ordinary staging continuation.
17414        std::fs::write(
17415            scratch.join("retained"),
17416            "upstream and retained resolution\n",
17417        )
17418        .unwrap();
17419        runner.state.resource_block = Some(crate::run::ResourceBlock {
17420            node: "rebase".into(),
17421            kind: "repository coordination".into(),
17422            detail: "busy".into(),
17423            at: Timestamp::now(),
17424            cause: crate::run::ResourceBlockCause::Other,
17425        });
17426        runner.state.save().unwrap();
17427        // Advance the remote without moving either checkout participating
17428        // in the standing rebase.
17429        let newer = git::git(
17430            &runner.state.repo,
17431            &[
17432                "commit-tree",
17433                "main^{tree}",
17434                "-p",
17435                "main",
17436                "-m",
17437                "base advanced during wait",
17438            ],
17439        )
17440        .await
17441        .unwrap();
17442        git::git(
17443            &runner.state.repo,
17444            &[
17445                "push",
17446                "origin",
17447                &format!("{}:refs/heads/main", newer.trim()),
17448            ],
17449        )
17450        .await
17451        .unwrap();
17452        runner.state = RunState::load(&runner.state.id).unwrap();
17453        runner.state.resource_block = None;
17454        runner.sync_to_base().await.unwrap();
17455        assert!(!runner.state.base_sync_blocked());
17456        let op = &runner.state.base_sync_recovery.operations[0];
17457        assert_eq!(op.head, head);
17458        assert_eq!(op.tip, tip);
17459        assert!(op.finished);
17460        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17461        assert_eq!(runner.state.rebase_fixes.len(), 1);
17462        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17463            .await
17464            .unwrap();
17465        assert_ne!(latest, tip);
17466        runner.state = RunState::load(&runner.state.id).unwrap();
17467        runner.sync_to_base().await.unwrap();
17468        assert!(!runner.state.base_sync_blocked());
17469        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17470        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17471        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17472    }
17473
17474    #[cfg(unix)]
17475    #[tokio::test]
17476    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17477        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17478        runner.state.reserve_base_operation(&head, &tip);
17479        runner.state.save().unwrap();
17480        std::fs::write(
17481            scratch.join("retained"),
17482            "upstream and retained resolution\n",
17483        )
17484        .unwrap();
17485        git::git(&scratch, &["add", "retained"]).await.unwrap();
17486        git::git(
17487            &scratch,
17488            &["-c", "core.editor=true", "rebase", "--continue"],
17489        )
17490        .await
17491        .unwrap();
17492        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17493        assert!(
17494            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17495                .await
17496                .unwrap()
17497                .is_empty()
17498        );
17499        assert_eq!(
17500            crate::rebase::finish(
17501                &mut runner.state,
17502                &scratch,
17503                "fix/retained",
17504                &head,
17505                &tip,
17506                &["retained".into()],
17507                ""
17508            )
17509            .await
17510            .unwrap(),
17511            crate::rebase::Rebased::Applied
17512        );
17513        assert!(!scratch.exists());
17514        // Simulate losing the caller before finish_base_operation/save.
17515        runner.state = RunState::load(&runner.state.id).unwrap();
17516        let op = &runner.state.base_sync_recovery.operations[0];
17517        assert!(!op.finished);
17518        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17519        runner.sync_to_base().await.unwrap();
17520        assert!(!runner.state.base_sync_blocked());
17521        assert!(runner.state.base_sync_recovery.operations[0].finished);
17522        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17523        assert_eq!(runner.state.rebase_fixes.len(), 1);
17524        let winner = runner.state.winner().unwrap().clone();
17525        let wait = runner.repository_wait();
17526        assert!(
17527            verify_current_head(&mut runner.state, &winner, &wait)
17528                .await
17529                .unwrap()
17530        );
17531        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17532    }
17533
17534    #[cfg(unix)]
17535    #[tokio::test]
17536    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17537        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17538            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17539            runner.state.reserve_base_operation(&head, &tip);
17540            runner.state.save().unwrap();
17541            std::fs::write(
17542                scratch.join("retained"),
17543                "upstream and retained resolution\n",
17544            )
17545            .unwrap();
17546            git::git(&scratch, &["add", "retained"]).await.unwrap();
17547            git::git(
17548                &scratch,
17549                &["-c", "core.editor=true", "rebase", "--continue"],
17550            )
17551            .await
17552            .unwrap();
17553            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17554            assert!(!git::rebase_in_progress(&scratch).await);
17555            assert!(
17556                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17557                    .await
17558                    .unwrap()
17559                    .is_empty()
17560            );
17561            match damage {
17562                Some("record") => {
17563                    let path =
17564                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17565                            .await
17566                            .unwrap();
17567                    std::fs::write(scratch.join(path), "").unwrap();
17568                }
17569                Some("foreign") => {
17570                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17571                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17572                    git::git(&scratch, &["commit", "-m", "foreign work"])
17573                        .await
17574                        .unwrap();
17575                }
17576                Some("dirty") => {
17577                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17578                }
17579                _ => {}
17580            }
17581            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17582            // Crash immediately after Git continuation, before finish runs.
17583            runner.state = RunState::load(&runner.state.id).unwrap();
17584            assert!(
17585                runner.state.base_sync_recovery.operations[0]
17586                    .validated_head
17587                    .is_none()
17588            );
17589            runner.sync_to_base().await.unwrap();
17590            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17591            assert_eq!(runner.state.base_episode_used(), 1);
17592            assert_eq!(runner.state.rebase_fixes.len(), 1);
17593            assert_eq!(
17594                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17595                retained_head
17596            );
17597            if damage.is_some() {
17598                assert!(runner.state.base_sync_blocked());
17599                assert!(scratch.exists());
17600                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17601                assert!(
17602                    runner.state.base_sync_recovery.operations[0]
17603                        .validated_head
17604                        .is_none()
17605                );
17606            } else {
17607                assert!(!runner.state.base_sync_blocked());
17608                assert!(!scratch.exists());
17609                assert!(runner.state.base_sync_recovery.operations[0].finished);
17610                assert_eq!(
17611                    runner.state.base_sync_recovery.operations[0]
17612                        .validated_head
17613                        .as_deref(),
17614                    Some(result.as_str())
17615                );
17616                let winner = runner.state.winner().unwrap().clone();
17617                let wait = runner.repository_wait();
17618                assert!(
17619                    verify_current_head(&mut runner.state, &winner, &wait)
17620                        .await
17621                        .unwrap()
17622                );
17623                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17624                runner.state = RunState::load(&runner.state.id).unwrap();
17625                runner.sync_to_base().await.unwrap();
17626                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17627            }
17628        }
17629    }
17630
17631    #[cfg(unix)]
17632    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17633        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17634        // This is an ordinary gate stop, not an authorized base-budget recovery.
17635        runner.state.prep_complete = Some(true);
17636        runner.state.base_sync = None;
17637        runner.state.base_sync_recovery = Default::default();
17638        runner.state.config.graph.gate_fix_rounds = 1;
17639        let mut review = review_round(true, 0, 1, 1, false, true);
17640        review.head = head.clone();
17641        review.verified_head = Some(head.clone());
17642        runner.state.reviews = vec![review];
17643        runner.state.gate = vec![CommandOutcome {
17644            command: "old gate".into(),
17645            code: Some(1),
17646            output_tail: "old failure".into(),
17647            duration_ms: 0,
17648            resource_blocked: false,
17649        }];
17650        runner.state.gate_head = Some(head.clone());
17651        runner.state.gate_ran = true;
17652        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17653            agent: "fixture".into(),
17654            failed: runner.state.gate.clone(),
17655            notes: "spent".into(),
17656            committed: false,
17657            error: None,
17658        }];
17659        runner.state.land_approval = Some(crate::run::LandApproval {
17660            question: "old approval".into(),
17661            head: head.clone(),
17662        });
17663        runner.state.status = RunStatus::Blocked;
17664        (tmp, runner, head)
17665    }
17666
17667    #[cfg(unix)]
17668    #[tokio::test]
17669    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17670        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17671            let (tmp, mut runner, old) = resumed_gate_fixture().await;
17672            let phases = tmp.path().join("phases");
17673            runner.state.config.verify.e2e = vec![format!(
17674                "echo e2e >> '{}'; {}",
17675                phases.display(),
17676                if e2e_ok {
17677                    "true"
17678                } else {
17679                    "echo new-e2e-failure; false"
17680                }
17681            )];
17682            runner.state.config.verify.gate = vec![format!(
17683                "echo gate >> '{}'; {}",
17684                phases.display(),
17685                if gate_ok {
17686                    "true"
17687                } else {
17688                    "echo new-gate-failure; false"
17689                }
17690            )];
17691            runner.state.save().unwrap();
17692            runner.state = RunState::load(&runner.state.id).unwrap();
17693            runner.sync_to_base().await.unwrap();
17694            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17695            assert_ne!(head, old);
17696            // Reentry after Git updated but before verification was invalidated.
17697            runner.state = RunState::load(&runner.state.id).unwrap();
17698            runner.execute_graph().await.unwrap();
17699            assert_eq!(runner.state.gate_fixes.len(), 1);
17700            assert_eq!(runner.state.reviews.len(), 1);
17701            assert_eq!(runner.state.reviews[0].head, old);
17702            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17703            assert_eq!(runner.state.verification_history.len(), 1);
17704            let history = &runner.state.verification_history[0];
17705            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17706            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17707            assert!(history.archived_at.is_some());
17708            assert_eq!(history.fixes_spent, Some(1));
17709            assert_eq!(history.gate[0].output_tail, "old failure");
17710            let archive_events: Vec<_> = runner
17711                .state
17712                .events
17713                .iter()
17714                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17715                .collect();
17716            assert_eq!(archive_events.len(), 1);
17717            assert!(archive_events[0].message.contains(&short(&old)));
17718            assert!(archive_events[0].message.contains(&short(&head)));
17719            assert!(
17720                archive_events[0]
17721                    .message
17722                    .contains("1 gate-fix round(s) already spent stay spent")
17723            );
17724            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17725            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17726            if e2e_ok {
17727                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17728                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17729            } else {
17730                assert!(!runner.state.gate_ran);
17731            }
17732            assert_eq!(
17733                runner.state.status,
17734                if e2e_ok && gate_ok {
17735                    RunStatus::Ready
17736                } else {
17737                    RunStatus::Blocked
17738                }
17739            );
17740            if !e2e_ok || !gate_ok {
17741                let cause = runner.state.verification_blocking_cause().unwrap();
17742                assert!(cause.contains(&head));
17743                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17744                // A later successful progress event must not become the cause.
17745                runner.state.event("land", "rebased successfully");
17746                let notice = crate::notices::run_ended(&runner.state).unwrap();
17747                assert!(!format!("{notice:?}").contains("rebased successfully"));
17748                assert!(runner.state.merge.is_none());
17749            }
17750            let before = std::fs::read_to_string(&phases).unwrap();
17751            runner.state.save().unwrap();
17752            runner.state = RunState::load(&runner.state.id).unwrap();
17753            runner.review_loop().await.unwrap();
17754            runner.gate().await.unwrap();
17755            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17756            assert_eq!(runner.state.verification_history.len(), 1);
17757            let history = &runner.state.verification_history[0];
17758            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17759            assert!(history.archived_at.is_some());
17760            assert_eq!(history.fixes_spent, Some(1));
17761        }
17762    }
17763
17764    #[cfg(unix)]
17765    #[tokio::test]
17766    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17767        let (tmp, mut runner, head) = resumed_gate_fixture().await;
17768        let marker = tmp.path().join("gate-ran");
17769        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17770        runner.review_loop().await.unwrap();
17771        runner.gate().await.unwrap();
17772        assert_eq!(runner.state.status, RunStatus::Blocked);
17773        assert!(!marker.exists());
17774        assert!(runner.state.verification_history.is_empty());
17775        runner.state.gate_head = None;
17776        runner.gate().await.unwrap();
17777        assert!(marker.exists());
17778        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17779        assert_eq!(runner.state.verification_history.len(), 1);
17780        assert_eq!(runner.state.gate_fixes.len(), 1);
17781    }
17782
17783    #[cfg(unix)]
17784    #[tokio::test]
17785    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17786        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17787        runner.sync_to_base().await.unwrap();
17788        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17789        runner.state.reconcile_verification_head(&head);
17790        // Both crash-before-command and resource wait are unresolved evidence.
17791        for outcomes in [
17792            Vec::new(),
17793            vec![CommandOutcome {
17794                command: "true".into(),
17795                code: None,
17796                output_tail: "cache unavailable".into(),
17797                duration_ms: 0,
17798                resource_blocked: true,
17799            }],
17800        ] {
17801            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17802                head: head.clone(),
17803                passed: false,
17804                outcomes,
17805            });
17806            runner.state.gate_ran = false;
17807            runner.state.gate.clear();
17808            runner.state.save().unwrap();
17809            runner.state = RunState::load(&runner.state.id).unwrap();
17810            runner.merge().await.unwrap();
17811            assert!(runner.state.merge.is_none());
17812            runner.review_loop().await.unwrap();
17813            runner.gate().await.unwrap();
17814            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17815            assert!(runner.state.gate_allows_progress());
17816            assert_eq!(runner.state.gate_fixes.len(), 1);
17817            assert!(runner.state.merge.is_none());
17818        }
17819        // Moving the winner after a successful gate cannot publish an unchecked SHA.
17820        git::git(
17821            &runner.state.repo,
17822            &["commit", "--allow-empty", "-m", "external update"],
17823        )
17824        .await
17825        .unwrap();
17826        runner.merge().await.unwrap();
17827        assert!(runner.state.merge.is_none());
17828        assert_eq!(runner.state.status, RunStatus::Blocked);
17829    }
17830
17831    #[cfg(unix)]
17832    #[tokio::test]
17833    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17834        for waiting_on_gate in [false, true] {
17835            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17836            runner.sync_to_base().await.unwrap();
17837            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17838            runner.state.reconcile_verification_head(&head);
17839            if waiting_on_gate {
17840                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17841                    head: head.clone(),
17842                    passed: true,
17843                    outcomes: Vec::new(),
17844                });
17845            }
17846            let cache = tmp.path().join("cache");
17847            runner.state.config.verify.e2e =
17848                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17849            runner.state.config.verify.gate = vec!["true".into()];
17850            runner.state.config.graph.timeout_resource_wait = 1;
17851            let owner =
17852                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
17853            let guard =
17854                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
17855                    crate::cache::AcquireOutcome::Acquired(g) => g,
17856                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
17857                };
17858            runner.review_loop().await.unwrap();
17859            runner.gate().await.unwrap();
17860            assert!(!runner.state.gate_ran);
17861            assert_eq!(runner.state.gate_fixes.len(), 1);
17862            assert!(runner.state.resource_block.is_some());
17863            runner.merge().await.unwrap();
17864            assert!(runner.state.merge.is_none());
17865            drop(guard);
17866            runner.state = RunState::load(&runner.state.id).unwrap();
17867            // The driver clears the previous pass's resource stop on resume.
17868            runner.state.resource_block = None;
17869            runner.review_loop().await.unwrap();
17870            runner.gate().await.unwrap();
17871            assert!(runner.state.gate_allows_progress());
17872            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17873            assert_eq!(runner.state.gate_fixes.len(), 1);
17874        }
17875    }
17876
17877    #[cfg(unix)]
17878    #[tokio::test]
17879    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
17880        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17881        runner.sync_to_base().await.unwrap();
17882        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17883        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
17884        // Verification passes before pre_gate and fails on the mechanically changed head.
17885        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
17886        runner.review_loop().await.unwrap();
17887        runner.gate().await.unwrap();
17888        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17889        assert_ne!(before, after);
17890        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
17891        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
17892        assert!(!runner.state.gate_allows_progress());
17893        runner.merge().await.unwrap();
17894        assert!(runner.state.merge.is_none());
17895        assert!(
17896            runner
17897                .state
17898                .verification_history
17899                .iter()
17900                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
17901        );
17902
17903        // An incomplete review is still an independent hold on a changed head.
17904        runner.state.reviews[0].clean = false;
17905        runner.state.reviews[0].answered = 0;
17906        runner.state.config.verify.e2e = vec!["true".into()];
17907        runner.review_loop().await.unwrap();
17908        runner.gate().await.unwrap();
17909        runner.merge().await.unwrap();
17910        assert_eq!(runner.state.status, RunStatus::Blocked);
17911        assert!(runner.state.merge.is_none());
17912        assert!(
17913            runner
17914                .state
17915                .verification_blocking_cause()
17916                .unwrap()
17917                .contains("review blocked")
17918        );
17919    }
17920
17921    #[cfg(unix)]
17922    #[tokio::test]
17923    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
17924        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
17925        runner.state.gate_head = None;
17926        runner.state.config.verify.gate =
17927            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
17928        runner.review_loop().await.unwrap();
17929        runner.gate().await.unwrap();
17930        assert_ne!(
17931            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17932            old
17933        );
17934        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
17935        assert!(!runner.state.gate_allows_progress());
17936        assert_eq!(runner.state.status, RunStatus::Blocked);
17937        runner.merge().await.unwrap();
17938        assert!(runner.state.merge.is_none());
17939    }
17940
17941    #[cfg(unix)]
17942    #[tokio::test]
17943    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
17944        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17945        let mut old_review = review_round(true, 0, 1, 1, false, true);
17946        old_review.head = head.clone();
17947        old_review.verified_head = Some(head.clone());
17948        runner.state.reviews = vec![old_review];
17949        runner.state.gate = vec![CommandOutcome {
17950            command: "old gate".into(),
17951            code: None,
17952            output_tail: "timed out after 2400s".into(),
17953            duration_ms: 2_400_000,
17954            resource_blocked: false,
17955        }];
17956        runner.state.gate_ran = true;
17957        runner.state.gate_head = Some(head.clone());
17958        runner.state.land_approval = Some(crate::run::LandApproval {
17959            question: "old-head-approval".into(),
17960            head: head.clone(),
17961        });
17962        runner.state.authorize_base_sync(&task).unwrap();
17963        runner.sync_to_base().await.unwrap();
17964        assert!(!runner.state.base_sync_blocked());
17965        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17966        assert_ne!(head, fresh);
17967        assert_eq!(runner.state.reviews.len(), 1);
17968        assert!(!runner.state.gate_ran);
17969        let saved = &runner.state.base_sync_recovery.authorizations[0];
17970        assert_eq!(saved.reviews[0].head, head);
17971        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
17972        assert!(saved.gate_ran);
17973        assert!(!saved.gate[0].ok());
17974        runner.review_loop().await.unwrap();
17975        runner.gate().await.unwrap();
17976        assert_eq!(runner.state.reviews[0].head, head);
17977        assert_eq!(
17978            runner.state.reviews[0].verified_head.as_deref(),
17979            Some(head.as_str())
17980        );
17981        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
17982        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17983        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
17984        assert!(runner.state.gate_ran);
17985        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
17986        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
17987        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
17988        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17989        assert_eq!(
17990            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
17991            "retained change"
17992        );
17993    }
17994
17995    #[cfg(unix)]
17996    #[tokio::test]
17997    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
17998        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17999        let queue = crate::queue::Queue::open();
18000        let mut owner = queue.get(&task).unwrap();
18001        owner.release_operator("test-resume", None);
18002        queue.put(&mut owner).unwrap();
18003        let generation = owner.resumption.as_ref().unwrap().generation;
18004        runner.state.task_instruction = Some(owner.instruction.clone());
18005        runner.state.driver_exited = true;
18006        runner.state.save().unwrap();
18007        let home = crate::run::home();
18008        let _claim = queue.claim(&task).unwrap();
18009        owner.resumption.as_mut().unwrap().at =
18010            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18011        queue.put(&mut owner).unwrap();
18012        assert!(
18013            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18014                .await
18015                .is_err(),
18016            "an approval from before the stop grants nothing"
18017        );
18018        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18019        queue.put(&mut owner).unwrap();
18020        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18021            .await
18022            .unwrap();
18023        assert!(
18024            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18025                .await
18026                .is_err()
18027        );
18028        runner.state = RunState::load(&runner.state.id).unwrap();
18029        assert_eq!(
18030            runner.state.base_sync_recovery.authorizations[0].generation,
18031            Some(generation)
18032        );
18033        runner.sync_to_base().await.unwrap();
18034        let repo = runner.state.repo.clone();
18035        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18036        git::git(&repo, &["checkout", "main"]).await.unwrap();
18037        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18038        git::git(&repo, &["add", "."]).await.unwrap();
18039        git::git(&repo, &["commit", "-m", "move base again"])
18040            .await
18041            .unwrap();
18042        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18043        git::git(&repo, &["checkout", "fix/retained"])
18044            .await
18045            .unwrap();
18046        runner.sync_to_base().await.unwrap();
18047        assert!(!runner.state.base_sync_blocked());
18048        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18049        assert_eq!(runner.state.base_episode_used(), 2);
18050        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18051        assert_ne!(fresh, head);
18052        assert!(!runner.state.base_recovery_pending());
18053        // Even replaying the generation with a later timestamp grants nothing.
18054        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18055        queue.put(&mut owner).unwrap();
18056        assert!(
18057            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18058                .await
18059                .is_err()
18060        );
18061        runner.state = RunState::load(&runner.state.id).unwrap();
18062        runner.sync_to_base().await.unwrap();
18063        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18064        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18065        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18066    }
18067
18068    #[cfg(unix)]
18069    #[tokio::test]
18070    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18071        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18072        runner.state.base_sync = None;
18073        runner.state.status = RunStatus::Blocked;
18074        let mut second = runner.state.candidates[0].clone();
18075        second.label = 'B';
18076        runner.state.candidates.push(second);
18077        assert!(runner.state.votes.is_empty());
18078        assert!(runner.state.deliberation.is_empty());
18079        runner.deliberate().await.unwrap();
18080        runner.vote().await.unwrap();
18081        assert_eq!(runner.state.status, RunStatus::Blocked);
18082        assert!(runner.state.votes.is_empty());
18083        assert!(runner.state.deliberation.is_empty());
18084    }
18085
18086    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18087    /// reads nothing else off it.
18088    fn runner_at(status: RunStatus) -> Runner {
18089        let mut state = RunState::new(
18090            PathBuf::from("/nonexistent/repo"),
18091            "main".to_owned(),
18092            "deadbeef".to_owned(),
18093            "task".to_owned(),
18094            Config::default(),
18095        );
18096        // These fixtures test execution flow, independent of host disk capacity.
18097        state.config.disk.min_free_bytes = 0;
18098        // These fixtures share one home and save concurrently. Short clock
18099        // IDs can collide, making an ownership test read another test's run.
18100        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18101        state.id = format!(
18102            "20000101-000000-{:04x}",
18103            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18104        );
18105        state.status = status;
18106        Runner {
18107            state,
18108            roles: ResolvedRoles {
18109                implementers: Vec::new(),
18110                judges: Vec::new(),
18111                reviewers: Vec::new(),
18112                fixer: None,
18113                conductor: conductor(),
18114                implementer_roster: Vec::new(),
18115                judge_roster: Vec::new(),
18116                reviewer_roster: Vec::new(),
18117            },
18118            sem: Arc::new(Semaphore::new(1)),
18119            pause: Pause::new(),
18120            interrupt: Pause::new(),
18121        }
18122    }
18123
18124    #[tokio::test]
18125    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18126        crate::run::pin_test_home();
18127        let mut runner = runner_at(RunStatus::Gating);
18128        runner.state.save().unwrap();
18129        let id = runner.state.id.clone();
18130        let passes = runner.state.passes.len();
18131        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18132        let mut claiming = Box::pin(runner.claim_driver());
18133        std::future::poll_fn(|cx| {
18134            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18135            std::task::Poll::Ready(())
18136        })
18137        .await;
18138        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18139        drop(owner);
18140        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18141            .await
18142            .unwrap()
18143            .unwrap();
18144        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18145        assert!(execution.runner.state.resource_block.is_none());
18146        assert_eq!(
18147            execution.runner.state.liveness(false),
18148            crate::run::Liveness::Live
18149        );
18150        drop(execution);
18151        assert!(RunState::load(&id).unwrap().driver_exited);
18152    }
18153
18154    #[tokio::test]
18155    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18156        crate::run::pin_test_home();
18157        let mut runner = runner_at(RunStatus::Gating);
18158        runner.state.save().unwrap();
18159        // run.json can commit, but the subsequent lease metadata write fails.
18160        // Cleanup must run after releasing the short ownership lock.
18161        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18162        assert!(runner.claim_driver().await.is_err());
18163        assert!(runner.state.driver_exited);
18164        assert_eq!(runner.state.driver_pid, None);
18165        let disk = RunState::load(&runner.state.id).unwrap();
18166        assert!(disk.driver_exited);
18167        assert_eq!(disk.driver_pid, None);
18168        assert_eq!(disk.status, RunStatus::Gating);
18169    }
18170
18171    #[tokio::test]
18172    async fn an_early_graph_error_releases_its_driver() {
18173        crate::run::pin_test_home();
18174        let mut runner = runner_at(RunStatus::Blocked);
18175        runner.state.config.graph.selected_implementer = Some("selected".into());
18176        runner.state.save().unwrap();
18177        assert!(runner.execute().await.is_err());
18178        let disk = RunState::load(&runner.state.id).unwrap();
18179        assert!(disk.driver_exited);
18180        assert_eq!(disk.driver_pid, None);
18181        assert_eq!(disk.status, RunStatus::Blocked);
18182    }
18183
18184    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18185        crate::run::MergeRepair {
18186            question: "repair-question".into(),
18187            head: "old-head".into(),
18188            pr: "https://github.com/o/r/pull/42".into(),
18189            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18190            gate: vec![outcome(Some(0), false)],
18191            gate_head: Some("old-head".into()),
18192            gate_fixes: Vec::new(),
18193            pre_gate: Vec::new(),
18194            phase,
18195            failure: None,
18196        }
18197    }
18198
18199    #[tokio::test]
18200    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18201        crate::run::pin_test_home();
18202        let mut runner = runner_at(RunStatus::Landing);
18203        runner.state.merge_repairs.push(merge_repair_request(
18204            crate::run::MergeRepairPhase::Requested,
18205        ));
18206        runner.state.save().unwrap();
18207        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18208        assert!(
18209            runner
18210                .execute()
18211                .await
18212                .unwrap_err()
18213                .to_string()
18214                .contains("already running")
18215        );
18216        assert_eq!(
18217            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18218            crate::run::MergeRepairPhase::Requested
18219        );
18220        drop(claim);
18221        let mut disk = runner.state.clone();
18222        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18223        disk.save().unwrap();
18224        assert!(
18225            runner
18226                .execute()
18227                .await
18228                .unwrap_err()
18229                .to_string()
18230                .contains("decision changed")
18231        );
18232        assert_eq!(
18233            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18234            crate::run::MergeRepairPhase::Complete
18235        );
18236    }
18237
18238    #[test]
18239    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18240        crate::run::pin_test_home();
18241        let mut runner = runner_at(RunStatus::Landing);
18242        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18243        runner.state.reviews = request.reviews.clone();
18244        runner.state.gate = request.gate.clone();
18245        runner.state.gate_ran = true;
18246        runner.state.land_approval = Some(crate::run::LandApproval {
18247            question: "old".into(),
18248            head: "old-head".into(),
18249        });
18250        runner.state.merge_repairs.push(request);
18251        runner.begin_merge_repair_review(0).unwrap();
18252        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18253        assert!(!runner.state.gate_status().ok());
18254        assert!(runner.state.land_approval.is_none());
18255        let disk = RunState::load(&runner.state.id).unwrap();
18256        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18257        assert!(disk.merge_repairs[0].gate[0].ok());
18258        assert_eq!(
18259            disk.merge_repairs[0].phase,
18260            crate::run::MergeRepairPhase::Reviewing
18261        );
18262        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18263        assert!(disk.reviews.is_empty());
18264    }
18265
18266    #[tokio::test]
18267    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18268        crate::run::pin_test_home();
18269        for phase in [
18270            crate::run::MergeRepairPhase::Fixing,
18271            crate::run::MergeRepairPhase::Requested,
18272        ] {
18273            let mut runner = runner_at(RunStatus::Landing);
18274            runner.state.config.graph.land_rounds = 0;
18275            runner.state.merge_repairs.push(merge_repair_request(phase));
18276            runner.state.save().unwrap();
18277            assert!(!runner.repair_merge_review().await.unwrap());
18278            assert_eq!(runner.state.status, RunStatus::Blocked);
18279            assert_eq!(
18280                runner.state.merge_repairs[0].phase,
18281                crate::run::MergeRepairPhase::Failed
18282            );
18283            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18284            assert!(
18285                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18286                    "interrupted"
18287                } else {
18288                    "budget exhausted"
18289                })
18290            );
18291            assert!(runner.state.merge.is_none());
18292        }
18293    }
18294
18295    /// `park_here` folding in the reason `Pause::park_because` recorded -
18296    /// this is what lets an operator reading a run's events tell an
18297    /// interrupt-driven park from an ordinary shutdown park.
18298    #[test]
18299    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18300        crate::run::pin_test_home();
18301        let mut runner = runner_at(RunStatus::Implementing);
18302        let interrupt = Pause::new();
18303        runner.watch_interrupt(interrupt.clone());
18304
18305        interrupt.park_because("task a1b2 asked to run first");
18306
18307        assert!(runner.park_here().expect("park_here"));
18308        assert!(runner.state.parked);
18309        let last = runner.state.events.last().expect("a park event");
18310        assert_eq!(last.node, "park");
18311        assert!(
18312            last.message.contains("task a1b2 asked to run first"),
18313            "expected the interrupt reason in {:?}",
18314            last.message
18315        );
18316    }
18317
18318    /// The cause is stored on the pass when it parks, and the first cause a
18319    /// shared pause was given is the one that stays.
18320    #[test]
18321    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18322        crate::run::pin_test_home();
18323        let mut runner = runner_at(RunStatus::Implementing);
18324        runner.state.begin_pass();
18325        let shutdown = Pause::new();
18326        runner.on_pause(shutdown.clone());
18327        let upgrade = ParkCause::Upgrade {
18328            from: "0.121.0".to_owned(),
18329            to: Some("0.121.1".to_owned()),
18330        };
18331        shutdown.park_for(upgrade.clone());
18332        shutdown.park_for(ParkCause::Operator);
18333
18334        assert!(runner.park_here().expect("park_here"));
18335        assert_eq!(
18336            runner.state.passes[0].end,
18337            Some(crate::run::PassEnd::Parked { cause: upgrade })
18338        );
18339        let last = runner.state.events.last().expect("a park event");
18340        assert!(last.message.contains("0.121.0"), "{}", last.message);
18341    }
18342
18343    #[tokio::test]
18344    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18345        crate::run::pin_test_home();
18346        let mut runner = runner_at(RunStatus::Reviewing);
18347        runner.state.save().unwrap();
18348        let mut execution = runner.claim_driver().await.expect("first execution");
18349        assert_eq!(execution.runner.state.passes.len(), 1);
18350        execution
18351            .driver
18352            .finish(&mut execution.runner.state)
18353            .await
18354            .unwrap();
18355    }
18356
18357    #[tokio::test]
18358    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18359        crate::run::pin_test_home();
18360        let mut runner = runner_at(RunStatus::Reviewing);
18361        runner.state.config.agents = vec![conductor()];
18362        runner.state.config.graph.timeout_resource_wait = 0;
18363        runner.state.save().unwrap();
18364        let id = runner.state.id.clone();
18365        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18366        assert!(runner.claim_driver().await.is_err());
18367        drop(owner);
18368        drop(runner);
18369        let mut resumed = Runner::resume(&id).unwrap();
18370        assert_eq!(resumed.state.driver_claimed, Some(false));
18371        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18372        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18373        let mut execution = resumed
18374            .claim_driver()
18375            .await
18376            .expect("retry initial claim after reload");
18377        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18378        assert_eq!(execution.runner.state.passes.len(), 1);
18379        execution
18380            .driver
18381            .finish(&mut execution.runner.state)
18382            .await
18383            .unwrap();
18384    }
18385
18386    #[tokio::test]
18387    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18388        crate::run::pin_test_home();
18389        let mut runner = runner_at(RunStatus::Reviewing);
18390        runner.state.config.agents = vec![conductor()];
18391        runner.state.save().unwrap();
18392        let id = runner.state.id.clone();
18393        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18394        let mut resumed = Runner::resume(&id).unwrap();
18395        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18396        assert!(resumed.claim_driver().await.is_err());
18397        assert!(RunState::load(&id).unwrap().passes.is_empty());
18398        drop(lease);
18399        let mut execution = resumed.claim_driver().await.unwrap();
18400        execution
18401            .driver
18402            .finish(&mut execution.runner.state)
18403            .await
18404            .unwrap();
18405    }
18406
18407    #[tokio::test]
18408    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18409        crate::run::pin_test_home();
18410        let mut runner = runner_at(RunStatus::Reviewing);
18411        let state = &mut runner.state;
18412        state.config.agents = vec![conductor()];
18413        state.driver_claimed = None;
18414        state.driver_pid = Some(std::process::id());
18415        state.driver_started_at = None;
18416        state.begin_pass();
18417        state.save().unwrap();
18418        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18419        std::fs::write(
18420            state.dir().join("driver.json"),
18421            serde_json::to_vec(&serde_json::json!({
18422                "pid": state.driver_pid, "started_at": null,
18423                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18424            }))
18425            .unwrap(),
18426        )
18427        .unwrap();
18428        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18429        drop(lease);
18430        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18431        let mut resumed = Runner::resume(&state.id).unwrap();
18432        assert!(!resumed.state.driver_exited);
18433        let mut execution = resumed
18434            .claim_driver()
18435            .await
18436            .expect("lease proves previous execution ended");
18437        assert_eq!(execution.runner.state.passes.len(), 2);
18438        execution
18439            .driver
18440            .finish(&mut execution.runner.state)
18441            .await
18442            .unwrap();
18443    }
18444
18445    #[tokio::test]
18446    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18447        crate::run::pin_test_home();
18448        let mut runner = runner_at(RunStatus::Reviewing);
18449        let mut json = serde_json::to_value(&runner.state).unwrap();
18450        json.as_object_mut().unwrap().remove("driver_claimed");
18451        runner.state = serde_json::from_value(json).unwrap();
18452        assert_eq!(runner.state.driver_claimed, None);
18453        runner.state.save().unwrap();
18454        let mut execution = runner
18455            .claim_driver()
18456            .await
18457            .expect("the execution lease, not a missing marker, excludes a second driver");
18458        assert_eq!(execution.runner.state.passes.len(), 1);
18459        execution
18460            .driver
18461            .finish(&mut execution.runner.state)
18462            .await
18463            .unwrap();
18464    }
18465
18466    #[tokio::test]
18467    async fn driver_admission_refuses_a_proven_live_driver() {
18468        crate::run::pin_test_home();
18469        let mut runner = runner_at(RunStatus::Reviewing);
18470        let pid = std::process::id();
18471        runner.state.driver_pid = Some(pid);
18472        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18473        assert!(runner.state.driver_started_at.is_some());
18474        runner.state.save().unwrap();
18475        let error = runner
18476            .claim_driver()
18477            .await
18478            .err()
18479            .expect("a live driver must refuse");
18480        assert!(error.to_string().contains("live driver"), "{error:#}");
18481        let saved = RunState::load(&runner.state.id).unwrap();
18482        assert_eq!(saved.driver_pid, Some(pid));
18483        assert_eq!(saved.status, RunStatus::Reviewing);
18484        assert!(saved.passes.is_empty());
18485    }
18486
18487    #[test]
18488    fn parking_resource_halt_persists_the_pass_once() {
18489        crate::run::pin_test_home();
18490        let mut runner = runner_at(RunStatus::Reviewing);
18491        runner.state.begin_pass();
18492        runner.state.resource_block = Some(crate::run::ResourceBlock {
18493            node: "fix".into(),
18494            kind: "shared build cache".into(),
18495            detail: "cancelled".into(),
18496            at: Timestamp::now(),
18497            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18498        });
18499        let cause = ParkCause::Upgrade {
18500            from: "old".into(),
18501            to: Some("new".into()),
18502        };
18503        runner.pause.park_for(cause.clone());
18504        // Both direct fixer exits and graph resource boundaries must complete
18505        // parking, rather than returning a stale Reviewing record.
18506        runner.stop_for_resource().unwrap();
18507        assert!(runner.resource_halt().unwrap());
18508        assert!(runner.park_here().unwrap());
18509        let saved = RunState::load(&runner.state.id).unwrap();
18510        assert!(saved.parked);
18511        assert_eq!(saved.status, RunStatus::Reviewing);
18512        assert!(saved.passes[0].ended_at.is_some());
18513        assert_eq!(
18514            saved.passes[0].end,
18515            Some(crate::run::PassEnd::Parked {
18516                cause: cause.clone()
18517            })
18518        );
18519        assert_eq!(
18520            saved.resource_block.unwrap().cause,
18521            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18522        );
18523        assert_eq!(
18524            saved
18525                .events
18526                .iter()
18527                .filter(|event| event.node == "park")
18528                .count(),
18529            1
18530        );
18531    }
18532
18533    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
18534    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
18535    /// cleared) must not make a *different* run - one only watching its own,
18536    /// unshared interrupt `Pause` - see itself as parked. If a future change
18537    /// ever collapsed these back into one handle, the interrupt scheduler
18538    /// would park every run for the rest of the daemon's life, not just the
18539    /// one it meant to interrupt.
18540    #[test]
18541    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18542        crate::run::pin_test_home();
18543        let mut runner = runner_at(RunStatus::Implementing);
18544        let shutdown = Pause::new();
18545        runner.on_pause(shutdown.clone());
18546        let interrupt = Pause::new();
18547        runner.watch_interrupt(interrupt.clone());
18548
18549        // Nobody has asked for anything yet.
18550        assert!(!runner.park_here().expect("park_here"));
18551        assert!(!runner.state.parked);
18552
18553        // Only the interrupt handle fires; the shutdown handle stays clear.
18554        interrupt.park_because("test");
18555        assert!(!shutdown.parked());
18556        assert!(runner.park_here().expect("park_here"));
18557    }
18558
18559    /// The property every prior attempt at this feature failed to pin down:
18560    /// asking a run to park while one of its nodes has a real, in-flight
18561    /// async operation running (an agent call, in production) must not cut
18562    /// that operation short. `park_here` is only ever consulted *between*
18563    /// `execute`'s node calls - see its own doc - so nothing inside a node
18564    /// can observe a park request until the node itself returns. This proves
18565    /// that structurally, with real `tokio` concurrency and a channel
18566    /// handshake (never a sleep, which would only prove "usually", not
18567    /// "cannot"): the "node" below reports that it has genuinely started,
18568    /// and only then is the park requested; the node still has to be told to
18569    /// finish before `park_here` is ever called, exactly mirroring every
18570    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
18571    /// in `execute`.
18572    #[tokio::test]
18573    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18574        crate::run::pin_test_home();
18575        let mut runner = runner_at(RunStatus::Implementing);
18576        let interrupt = Pause::new();
18577        runner.watch_interrupt(interrupt.clone());
18578
18579        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18580        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18581
18582        // Stands in for one node's in-flight agent call: it proves it has
18583        // genuinely started, then blocks - exactly as a spawned CLI process
18584        // does - until told to finish.
18585        let node = async move {
18586            started_tx.send(()).expect("send started");
18587            finish_rx.await.expect("recv finish");
18588            "node finished"
18589        };
18590
18591        let interrupter = async move {
18592            started_rx.await.expect("recv started");
18593            // The call is now genuinely in flight. Ask it to park.
18594            interrupt.park_because("higher-priority task waiting");
18595            // Nothing the node does can observe this yet - there is no
18596            // check inside it, by construction - so let the executor run
18597            // anything pending and then let the node finish on its own.
18598            tokio::task::yield_now().await;
18599            finish_tx.send(()).expect("send finish");
18600        };
18601
18602        let (node_result, ()) = tokio::join!(node, interrupter);
18603        assert_eq!(
18604            node_result, "node finished",
18605            "the in-flight call ran to completion"
18606        );
18607
18608        // Only now, at the boundary the real `execute` would check right
18609        // after this node, does the park take effect.
18610        assert!(runner.park_here().expect("park_here"));
18611        assert!(runner.state.parked);
18612    }
18613
18614    /// A run parked mid-competition carries every field it had accumulated
18615    /// through the exact same disk round-trip an ordinary resume uses -
18616    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
18617    /// Nothing about parking for an interrupt is a special case of that path;
18618    /// this is what proves it rather than assuming it.
18619    #[test]
18620    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18621        crate::run::pin_test_home();
18622        let mut runner = runner_at(RunStatus::Judging);
18623        // `Runner::resume` re-resolves roles from the saved config, which
18624        // refuses an empty roster - give it the same minimal one `conductor`
18625        // itself uses.
18626        runner.state.config.agents = vec![conductor()];
18627        runner.state.candidates = vec![Candidate {
18628            index: 0,
18629            label: 'A',
18630            agent: "alpha".to_owned(),
18631            branch: "magi/x/A".to_owned(),
18632            worktree: PathBuf::from("/nonexistent/worktree"),
18633            summary: "did the thing".to_owned(),
18634            stat: "1 file changed".to_owned(),
18635            files: 1,
18636            commits: 1,
18637            empty: false,
18638            failed: None,
18639            verified_noop: None,
18640            duration_ms: 1234,
18641            folded: false,
18642        }];
18643        let run_id = runner.state.id.clone();
18644
18645        let interrupt = Pause::new();
18646        runner.watch_interrupt(interrupt.clone());
18647        interrupt.park_because("task c3d4 asked to run first");
18648        assert!(runner.park_here().expect("park_here"));
18649
18650        let resumed = Runner::resume(&run_id).expect("resume");
18651        assert_eq!(resumed.state.candidates.len(), 1);
18652        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18653        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18654        assert_eq!(resumed.state.status, runner.state.status);
18655        assert!(
18656            resumed.state.parked,
18657            "still parked until `execute` actually walks the graph again"
18658        );
18659        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18660    }
18661
18662    /// A fresh open question on `run`, stored and handed back for assertions.
18663    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18664        let mut q = ask::Question::new(
18665            run.to_owned(),
18666            "implement".to_owned(),
18667            "impl-A".to_owned(),
18668            "Which storage backend should the cache use?".to_owned(),
18669            String::new(),
18670            vec!["SQLite".to_owned(), "Redis".to_owned()],
18671        );
18672        store.put(&mut q).unwrap();
18673        q
18674    }
18675
18676    #[test]
18677    fn a_failed_runs_open_question_is_abandoned() {
18678        ask_test_home();
18679        let store = ask::Questions::open();
18680        let mut runner = runner_at(RunStatus::Failed);
18681        let run = runner.state.id.clone();
18682        let q = ask_open_question(&store, &run);
18683
18684        runner.settle_questions();
18685
18686        let back = store.get(&q.id).unwrap();
18687        assert!(
18688            !back.status.open(),
18689            "the seat that asked died with the run; nobody is left to read an answer"
18690        );
18691        assert!(
18692            back.detail.contains(&run) && back.detail.contains("failed"),
18693            "the reason names what the run became, not just that it is gone: {}",
18694            back.detail
18695        );
18696    }
18697
18698    #[test]
18699    fn a_merged_runs_open_question_is_abandoned_too() {
18700        ask_test_home();
18701        let store = ask::Questions::open();
18702        // A run that finishes cleanly still leaves nobody to read an answer -
18703        // this is not only a failure-path cleanup.
18704        for status in [RunStatus::Merged, RunStatus::Ready] {
18705            let mut runner = runner_at(status);
18706            let run = runner.state.id.clone();
18707            let q = ask_open_question(&store, &run);
18708
18709            runner.settle_questions();
18710
18711            let back = store.get(&q.id).unwrap();
18712            assert!(
18713                !back.status.open(),
18714                "{status:?} run's question must not outlive the run"
18715            );
18716        }
18717    }
18718
18719    #[test]
18720    fn a_still_resumable_runs_open_question_is_left_alone() {
18721        ask_test_home();
18722        let store = ask::Questions::open();
18723        // `Blocked` and `Stalled` can still be resumed — the candidates, the
18724        // review round and the seat sessions are all still on disk — so a
18725        // question asked mid-round may yet get a real answer from a real
18726        // resume. Sweeping it here would be exactly the failure mode this
18727        // whole feature exists to avoid on the other side.
18728        for status in [RunStatus::Blocked, RunStatus::Stalled] {
18729            let mut runner = runner_at(status);
18730            let run = runner.state.id.clone();
18731            let q = ask_open_question(&store, &run);
18732
18733            runner.settle_questions();
18734
18735            let back = store.get(&q.id).unwrap();
18736            assert!(
18737                back.status.open(),
18738                "{status:?} is still alive; the question must still be waiting"
18739            );
18740        }
18741    }
18742
18743    #[test]
18744    fn settle_questions_never_touches_an_already_answered_question() {
18745        ask_test_home();
18746        let store = ask::Questions::open();
18747        let mut runner = runner_at(RunStatus::Failed);
18748        let run = runner.state.id.clone();
18749        let mut q = ask_open_question(&store, &run);
18750        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
18751            .unwrap();
18752        store.put(&mut q).unwrap();
18753
18754        // Called twice, the way a crash-recovered daemon reclaim and the
18755        // graph's own cleanup both can for the same run — `abandon_for_run`
18756        // only ever touches what is still open, so this must be inert both
18757        // times, not merely the second.
18758        runner.settle_questions();
18759        runner.settle_questions();
18760
18761        let back = store.get(&q.id).unwrap();
18762        assert_eq!(
18763            back.status,
18764            ask::QuestionStatus::Answered,
18765            "a real answer is a decision on record, never overwritten by a sweep"
18766        );
18767    }
18768
18769    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
18770    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
18771    /// without merging, whose winner is still the operator's answer to read.
18772    /// Nothing previously called `fold_run` itself with a real `tally`, so
18773    /// this is the first test to pin down the one distinction the whole
18774    /// automatic-fold feature depends on: the winner's worktree and branch
18775    /// must survive, everything else sharing the run's worktree bay - a
18776    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
18777    /// second sweep treats every non-winner directory under the bay alike -
18778    /// must not.
18779    #[tokio::test]
18780    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
18781        crate::run::pin_test_home();
18782        let tmp = tempfile::tempdir().expect("tempdir");
18783        let repo = tmp.path().join("repo");
18784        std::fs::create_dir_all(&repo).unwrap();
18785        init_repo(&repo);
18786
18787        let mut config = Config::default();
18788        config.graph.worktree_root = Some(tmp.path().join("wt"));
18789
18790        let mut state = RunState::new(
18791            repo.clone(),
18792            "main".to_owned(),
18793            "deadbeef".to_owned(),
18794            "task".to_owned(),
18795            config,
18796        );
18797        let root = state.worktree_root();
18798        let wt_a = root.join("cand-A");
18799        let wt_b = root.join("cand-B");
18800        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
18801            .await
18802            .expect("worktree A");
18803        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
18804            .await
18805            .expect("worktree B");
18806
18807        state.candidates = vec![
18808            Candidate {
18809                index: 0,
18810                label: 'A',
18811                agent: "alpha".to_owned(),
18812                branch: "magi/x/A".to_owned(),
18813                worktree: wt_a.clone(),
18814                summary: String::new(),
18815                stat: String::new(),
18816                files: 0,
18817                commits: 0,
18818                empty: false,
18819                failed: None,
18820                verified_noop: None,
18821                duration_ms: 0,
18822                folded: false,
18823            },
18824            Candidate {
18825                index: 1,
18826                label: 'B',
18827                agent: "beta".to_owned(),
18828                branch: "magi/x/B".to_owned(),
18829                worktree: wt_b.clone(),
18830                summary: String::new(),
18831                stat: String::new(),
18832                files: 0,
18833                commits: 0,
18834                empty: false,
18835                failed: None,
18836                verified_noop: None,
18837                duration_ms: 0,
18838                folded: false,
18839            },
18840        ];
18841        state.tally = Some(Tally {
18842            first_choice: BTreeMap::from([('A', 1)]),
18843            borda: BTreeMap::new(),
18844            winner: 'A',
18845            rankings: 1,
18846            unanimous_initial: true,
18847            deliberated: false,
18848            changed_votes: 0,
18849            unanimous_final: true,
18850            tie_break: None,
18851            judges: 1,
18852            present: 1,
18853            quorum: 1,
18854            met_quorum: true,
18855            uncontested: None,
18856        });
18857        state.status = RunStatus::Ready;
18858
18859        fold_run(&mut state, false, &crate::run::home())
18860            .await
18861            .expect("fold_run");
18862
18863        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
18864        assert!(
18865            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
18866            "the unmerged winner's branch survives"
18867        );
18868        assert!(
18869            !state.candidates[0].folded,
18870            "the winner is not marked folded"
18871        );
18872
18873        assert!(!wt_b.exists(), "the loser's worktree is removed");
18874        assert!(
18875            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
18876            "the loser's branch is removed"
18877        );
18878        assert!(state.candidates[1].folded, "the loser is marked folded");
18879    }
18880
18881    #[tokio::test]
18882    #[cfg(unix)]
18883    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
18884        use std::os::fd::AsRawFd as _;
18885        use std::time::Duration;
18886
18887        let tmp = tempfile::tempdir().unwrap();
18888        let repo = tmp.path().join("repo");
18889        std::fs::create_dir_all(&repo).unwrap();
18890        init_repo(&repo);
18891        let common = repo.join(".git");
18892        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18893        let home = tmp.path().join("home");
18894        let mut config = Config::default();
18895        config.coordination = crate::config::test_coordination();
18896        config.graph.worktree_root = Some(tmp.path().join("wt"));
18897        config.graph.timeout_resource_wait = 1;
18898        let mut state = RunState::new(
18899            repo.clone(),
18900            "main".into(),
18901            "deadbeef".into(),
18902            "task".into(),
18903            config,
18904        );
18905        let root = state.worktree_root();
18906        let candidate = root.join("cand-A");
18907        let reviewer = root.join("review-1");
18908        let branch = state.branch_for('A');
18909        git::worktree_add_branch(&repo, &candidate, &branch, "main")
18910            .await
18911            .unwrap();
18912        git::worktree_add_detached(&repo, &reviewer, "main")
18913            .await
18914            .unwrap();
18915        git::acquire_worktree_config(&repo).await.unwrap();
18916        state.enabled_worktree_config = true;
18917        state.status = RunStatus::Ready;
18918        state.candidates.push(Candidate {
18919            index: 0,
18920            label: 'A',
18921            agent: "implement".into(),
18922            branch: branch.clone(),
18923            worktree: candidate.clone(),
18924            summary: String::new(),
18925            stat: String::new(),
18926            files: 0,
18927            commits: 0,
18928            empty: false,
18929            failed: None,
18930            verified_noop: None,
18931            duration_ms: 0,
18932            folded: false,
18933        });
18934        state.save_under(&home).unwrap();
18935        let config_before = std::fs::read(common.join("config")).unwrap();
18936        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
18937            .await
18938            .unwrap();
18939        // A separate OS handle bypasses the launcher's local gate, exactly as
18940        // an external updater does, and must still prevent all folding work.
18941        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18942        assert_eq!(
18943            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18944            0
18945        );
18946        let mut fold = Box::pin(fold_run(&mut state, true, &home));
18947        assert!(
18948            tokio::time::timeout(Duration::from_millis(200), &mut fold)
18949                .await
18950                .is_err()
18951        );
18952        assert!(candidate.exists());
18953        assert!(reviewer.exists());
18954        assert!(git::branch_exists(&repo, &branch).await.unwrap());
18955        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
18956        assert_eq!(
18957            git::git(&repo, &["worktree", "list", "--porcelain"])
18958                .await
18959                .unwrap(),
18960            worktrees_before
18961        );
18962        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
18963            .await
18964            .unwrap()
18965            .unwrap_err();
18966        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
18967        drop(fold);
18968        assert!(candidate.exists() && reviewer.exists());
18969        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
18970        assert_eq!(
18971            state.resource_block.as_ref().unwrap().kind,
18972            "repository coordination"
18973        );
18974        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
18975        drop(external);
18976        let mut fold = Box::pin(fold_run(&mut state, true, &home));
18977        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
18978            .await
18979            .unwrap()
18980            .unwrap();
18981        drop(fold);
18982        assert!(removed.contains(&branch));
18983        assert!(!root.exists());
18984        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
18985        assert!(state.candidates[0].folded);
18986        assert!(!state.enabled_worktree_config);
18987        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
18988        let enabled = git::git_raw(
18989            &repo,
18990            &["config", "--local", "--get", "extensions.worktreeConfig"],
18991        )
18992        .await
18993        .unwrap();
18994        assert!(!enabled.ok(), "the last config reference was released");
18995    }
18996
18997    /// A branch handed to a later run is that run's (and its pull request's):
18998    /// folding the run that released it must not delete it.
18999    #[tokio::test]
19000    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19001        let tmp = tempfile::tempdir().expect("tempdir");
19002        let repo = tmp.path().join("repo");
19003        std::fs::create_dir_all(&repo).unwrap();
19004        init_repo(&repo);
19005        let home = tmp.path().join("home");
19006
19007        let mut config = Config::default();
19008        config.graph.worktree_root = Some(tmp.path().join("wt"));
19009        let mut state = RunState::new(
19010            repo.clone(),
19011            "main".to_owned(),
19012            "deadbeef".to_owned(),
19013            "task".to_owned(),
19014            config,
19015        );
19016        // The worktree is already gone (released); the branch survives.
19017        git::git(&repo, &["branch", "magi/x/A", "main"])
19018            .await
19019            .expect("branch");
19020        state.candidates = vec![Candidate {
19021            index: 0,
19022            label: 'A',
19023            agent: "alpha".to_owned(),
19024            branch: "magi/x/A".to_owned(),
19025            worktree: state.worktree_root().join("cand-A"),
19026            summary: String::new(),
19027            stat: String::new(),
19028            files: 0,
19029            commits: 0,
19030            empty: false,
19031            failed: None,
19032            verified_noop: None,
19033            duration_ms: 0,
19034            folded: true,
19035        }];
19036        state.released_to = Some("20260901-000000-new1".to_owned());
19037        state.released_branches = vec!["magi/x/A".to_owned()];
19038
19039        fold_run(&mut state, true, &home).await.expect("fold_run");
19040        state.status = RunStatus::Merged;
19041        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19042        state.merge = Some(crate::run::MergeOutcome {
19043            mode: MergeMode::Pr,
19044            ok: true,
19045            detail: "confirmed merged".into(),
19046            empty: false,
19047        });
19048        fold_run(&mut state, true, &home)
19049            .await
19050            .expect("merged fold");
19051
19052        assert!(
19053            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19054            "the handed-over branch survives a fold"
19055        );
19056    }
19057
19058    /// A winner with nothing ahead of the base is caught before `gh` is ever
19059    /// asked for a pull request, and the message carries what the task's
19060    /// references resolved to.
19061    #[tokio::test]
19062    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19063        let tmp = tempfile::tempdir().expect("tempdir");
19064        let repo = tmp.path().join("repo");
19065        std::fs::create_dir_all(&repo).unwrap();
19066        init_repo(&repo);
19067        let run = |args: &[&str]| {
19068            let out = std::process::Command::new("git")
19069                .quiet()
19070                .args(args)
19071                .current_dir(&repo)
19072                .output()
19073                .expect("spawn git");
19074            assert!(out.status.success(), "git {args:?}");
19075        };
19076        run(&["branch", "magi/x/A"]);
19077        run(&["checkout", "-q", "-b", "magi/x/B"]);
19078        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19079        run(&["add", "-A"]);
19080        run(&["commit", "-q", "-m", "work"]);
19081        run(&["checkout", "-q", "main"]);
19082
19083        // A pull request is compared against the remote's base, so the
19084        // fixture needs one. Before it exists nothing can be read, and the
19085        // answer must be "not empty".
19086        let probe = RunState::new(
19087            repo.clone(),
19088            "main".to_owned(),
19089            "deadbeef".to_owned(),
19090            "task".to_owned(),
19091            Config::default(),
19092        );
19093        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19094        let bare = tmp.path().join("origin.git");
19095        let out = std::process::Command::new("git")
19096            .quiet()
19097            .args(["init", "-q", "--bare"])
19098            .arg(&bare)
19099            .output()
19100            .expect("spawn git");
19101        assert!(out.status.success(), "git init --bare");
19102        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19103        run(&["push", "-q", "origin", "main"]);
19104
19105        let mut state = RunState::new(
19106            repo.clone(),
19107            "main".to_owned(),
19108            "deadbeef".to_owned(),
19109            "task".to_owned(),
19110            Config::default(),
19111        );
19112        state.seeds = vec![refs::Seed {
19113            token: "magi/27b2/A".to_owned(),
19114            kind: refs::SeedKind::Unresolved,
19115            sha: String::new(),
19116            branch: true,
19117            detail: "no branch or commit named magi/27b2/A".to_owned(),
19118        }];
19119
19120        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19121        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19122        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19123        let detail = empty_candidate_detail(&state, "main");
19124        assert!(detail.starts_with("empty candidate"), "{detail}");
19125        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19126        assert!(detail.contains("--start-from"), "{detail}");
19127    }
19128
19129    /// `status == Ready` used to be read as "this is the harmless
19130    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19131    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19132    /// run's PR was closed without merging — and reentering `merge` with
19133    /// `mode` still `Pr` does not know the difference, so it pushed and
19134    /// opened a second pull request. `mode == Local` reproduces the same
19135    /// blind spot without a network call: reentry must not attempt another
19136    /// git merge once this node has already recorded an outcome.
19137    #[tokio::test]
19138    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19139        let tmp = tempfile::tempdir().expect("tempdir");
19140        let repo = tmp.path().join("repo");
19141        std::fs::create_dir_all(&repo).unwrap();
19142        init_repo(&repo);
19143
19144        let mut config = Config::default();
19145        config.merge.mode = MergeMode::Local;
19146
19147        let mut state = RunState::new(
19148            repo.clone(),
19149            "main".to_owned(),
19150            "deadbeef".to_owned(),
19151            "task".to_owned(),
19152            config,
19153        );
19154        state.candidates = vec![Candidate {
19155            index: 0,
19156            label: 'A',
19157            agent: "alpha".to_owned(),
19158            branch: "does-not-exist".to_owned(),
19159            worktree: repo.clone(),
19160            summary: String::new(),
19161            stat: String::new(),
19162            files: 0,
19163            commits: 0,
19164            empty: false,
19165            failed: None,
19166            verified_noop: None,
19167            duration_ms: 0,
19168            folded: false,
19169        }];
19170        state.tally = Some(Tally {
19171            first_choice: BTreeMap::from([('A', 1)]),
19172            borda: BTreeMap::new(),
19173            winner: 'A',
19174            rankings: 1,
19175            unanimous_initial: true,
19176            deliberated: false,
19177            changed_votes: 0,
19178            unanimous_final: true,
19179            tie_break: None,
19180            judges: 0,
19181            present: 0,
19182            quorum: 0,
19183            met_quorum: true,
19184            uncontested: Some("only candidate A produced a change".to_owned()),
19185        });
19186        state.reviews = vec![ReviewRound {
19187            round: 1,
19188            head: "deadbeef".to_owned(),
19189            verified_head: None,
19190            verified_at: None,
19191            reviews: Vec::new(),
19192            e2e: Vec::new(),
19193            fix: None,
19194            blocking: 0,
19195            answered: 0,
19196            expected: 0,
19197            clean: true,
19198            verify_retried: false,
19199            e2e_deferred: false,
19200            e2e_defer_reason: None,
19201            progressed: false,
19202            vote_split: false,
19203            reconsideration: Vec::new(),
19204            verdict: None,
19205        }];
19206        state.gate = vec![CommandOutcome {
19207            command: "test".to_owned(),
19208            code: Some(0),
19209            output_tail: String::new(),
19210            duration_ms: 0,
19211            resource_blocked: false,
19212        }];
19213        state.gate_ran = true;
19214        // Reached its conclusion already — e.g. `land` closing the PR without
19215        // merging it, which (like the honest `MergeMode::None` path) leaves
19216        // `status` at `Ready`. The recorded outcome is what actually marks
19217        // this node done.
19218        state.status = RunStatus::Ready;
19219        state.merge = Some(MergeOutcome {
19220            mode: MergeMode::Local,
19221            ok: false,
19222            detail: "already concluded".to_owned(),
19223            empty: false,
19224        });
19225
19226        let mut runner = Runner {
19227            state,
19228            roles: ResolvedRoles {
19229                implementers: Vec::new(),
19230                judges: Vec::new(),
19231                reviewers: Vec::new(),
19232                fixer: None,
19233                conductor: conductor(),
19234                implementer_roster: Vec::new(),
19235                judge_roster: Vec::new(),
19236                reviewer_roster: Vec::new(),
19237            },
19238            sem: Arc::new(Semaphore::new(1)),
19239            pause: Pause::new(),
19240            interrupt: Pause::new(),
19241        };
19242
19243        runner.merge().await.expect("merge");
19244
19245        assert_eq!(
19246            runner.state.status,
19247            RunStatus::Ready,
19248            "a concluded run's status must not change on reentry"
19249        );
19250        assert_eq!(
19251            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19252            Some("already concluded"),
19253            "merge must not run again once the node already recorded an outcome"
19254        );
19255    }
19256
19257    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19258    /// when its last attempt was resource-blocked (the shared build cache
19259    /// could not be acquired or confirmed fresh in time - see
19260    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19261    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19262    /// a run merge a tree the gate never actually checked - exactly the case
19263    /// a contended cache produces on every retry until it clears. `merge`
19264    /// must refuse until `gate` has actually recorded an attempt.
19265    #[tokio::test]
19266    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19267        let tmp = tempfile::tempdir().expect("tempdir");
19268        let repo = tmp.path().join("repo");
19269        std::fs::create_dir_all(&repo).unwrap();
19270        init_repo(&repo);
19271
19272        let mut config = Config::default();
19273        config.merge.mode = MergeMode::Local;
19274
19275        let mut state = RunState::new(
19276            repo.clone(),
19277            "main".to_owned(),
19278            "deadbeef".to_owned(),
19279            "task".to_owned(),
19280            config,
19281        );
19282        state.candidates = vec![Candidate {
19283            index: 0,
19284            label: 'A',
19285            agent: "alpha".to_owned(),
19286            branch: "does-not-exist".to_owned(),
19287            worktree: repo.clone(),
19288            summary: String::new(),
19289            stat: String::new(),
19290            files: 0,
19291            commits: 0,
19292            empty: false,
19293            failed: None,
19294            verified_noop: None,
19295            duration_ms: 0,
19296            folded: false,
19297        }];
19298        state.tally = Some(Tally {
19299            first_choice: BTreeMap::from([('A', 1)]),
19300            borda: BTreeMap::new(),
19301            winner: 'A',
19302            rankings: 1,
19303            unanimous_initial: true,
19304            deliberated: false,
19305            changed_votes: 0,
19306            unanimous_final: true,
19307            tie_break: None,
19308            judges: 0,
19309            present: 0,
19310            quorum: 0,
19311            met_quorum: true,
19312            uncontested: Some("only candidate A produced a change".to_owned()),
19313        });
19314        state.reviews = vec![ReviewRound {
19315            round: 1,
19316            head: "deadbeef".to_owned(),
19317            verified_head: None,
19318            verified_at: None,
19319            reviews: Vec::new(),
19320            e2e: Vec::new(),
19321            fix: None,
19322            blocking: 0,
19323            answered: 0,
19324            expected: 0,
19325            clean: true,
19326            verify_retried: false,
19327            e2e_deferred: false,
19328            e2e_defer_reason: None,
19329            progressed: false,
19330            vote_split: false,
19331            reconsideration: Vec::new(),
19332            verdict: None,
19333        }];
19334        // The point: `gate` has not recorded anything yet.
19335        state.gate = Vec::new();
19336        state.gate_ran = false;
19337        state.status = RunStatus::Gating;
19338
19339        let mut runner = Runner {
19340            state,
19341            roles: ResolvedRoles {
19342                implementers: Vec::new(),
19343                judges: Vec::new(),
19344                reviewers: Vec::new(),
19345                fixer: None,
19346                conductor: conductor(),
19347                implementer_roster: Vec::new(),
19348                judge_roster: Vec::new(),
19349                reviewer_roster: Vec::new(),
19350            },
19351            sem: Arc::new(Semaphore::new(1)),
19352            pause: Pause::new(),
19353            interrupt: Pause::new(),
19354        };
19355
19356        runner.merge().await.expect("merge");
19357
19358        assert!(
19359            runner.state.merge.is_none(),
19360            "an empty gate must never be read as a passing one: {:?}",
19361            runner.state.merge
19362        );
19363    }
19364
19365    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19366    /// commands configured and `merge.mode` is `none` (a review-only run).
19367    /// `gate` must still record a real attempt — zero commands, vacuously
19368    /// passed — rather than leaving `state.gate` empty in a way `merge`
19369    /// cannot tell apart from "never ran"; otherwise the run reaches
19370    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19371    #[tokio::test]
19372    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19373        let tmp = tempfile::tempdir().expect("tempdir");
19374        let repo = tmp.path().join("repo");
19375        std::fs::create_dir_all(&repo).unwrap();
19376        init_repo(&repo);
19377
19378        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19379        let config = Config::default();
19380
19381        let mut state = RunState::new(
19382            repo.clone(),
19383            "main".to_owned(),
19384            "deadbeef".to_owned(),
19385            "task".to_owned(),
19386            config,
19387        );
19388        state.candidates = vec![Candidate {
19389            index: 0,
19390            label: 'A',
19391            agent: "alpha".to_owned(),
19392            branch: "does-not-exist".to_owned(),
19393            worktree: repo.clone(),
19394            summary: String::new(),
19395            stat: String::new(),
19396            files: 0,
19397            commits: 0,
19398            empty: false,
19399            failed: None,
19400            verified_noop: None,
19401            duration_ms: 0,
19402            folded: false,
19403        }];
19404        state.tally = Some(Tally {
19405            first_choice: BTreeMap::from([('A', 1)]),
19406            borda: BTreeMap::new(),
19407            winner: 'A',
19408            rankings: 1,
19409            unanimous_initial: true,
19410            deliberated: false,
19411            changed_votes: 0,
19412            unanimous_final: true,
19413            tie_break: None,
19414            judges: 0,
19415            present: 0,
19416            quorum: 0,
19417            met_quorum: true,
19418            uncontested: Some("only candidate A produced a change".to_owned()),
19419        });
19420        state.reviews = vec![ReviewRound {
19421            round: 1,
19422            head: "deadbeef".to_owned(),
19423            verified_head: None,
19424            verified_at: None,
19425            reviews: Vec::new(),
19426            e2e: Vec::new(),
19427            fix: None,
19428            blocking: 0,
19429            answered: 0,
19430            expected: 0,
19431            clean: true,
19432            verify_retried: false,
19433            e2e_deferred: false,
19434            e2e_defer_reason: None,
19435            progressed: false,
19436            vote_split: false,
19437            reconsideration: Vec::new(),
19438            verdict: None,
19439        }];
19440
19441        let mut runner = Runner {
19442            state,
19443            roles: ResolvedRoles {
19444                implementers: Vec::new(),
19445                judges: Vec::new(),
19446                reviewers: Vec::new(),
19447                fixer: None,
19448                conductor: conductor(),
19449                implementer_roster: Vec::new(),
19450                judge_roster: Vec::new(),
19451                reviewer_roster: Vec::new(),
19452            },
19453            sem: Arc::new(Semaphore::new(1)),
19454            pause: Pause::new(),
19455            interrupt: Pause::new(),
19456        };
19457
19458        runner.gate().await.expect("gate");
19459        assert!(
19460            runner.state.gate_ran,
19461            "zero configured commands is still a real attempt, not an unrun gate"
19462        );
19463        assert!(runner.state.gate.is_empty());
19464        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19465        assert_ne!(
19466            runner.state.status,
19467            RunStatus::Blocked,
19468            "a gate with nothing to check must not read as failed"
19469        );
19470
19471        runner.merge().await.expect("merge");
19472        assert_eq!(
19473            runner.state.status,
19474            RunStatus::Ready,
19475            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19476        );
19477    }
19478
19479    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19480    /// `verify.gate` (so the e2e leg and the final gate never build against
19481    /// different directories). With zero `verify.gate` commands but a
19482    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19483    /// that lease before discovering it had nothing to run - so a repo with
19484    /// no gate commands could come back `resource_blocked` (and therefore
19485    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19486    /// cache, exactly the contention this run's own zero commands could
19487    /// never have touched. `gate` must recognise there is nothing to check
19488    /// before it ever asks for the lease.
19489    #[tokio::test]
19490    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19491        crate::run::pin_test_home();
19492        let home = crate::run::home();
19493
19494        let tmp = tempfile::tempdir().expect("tempdir");
19495        let repo = tmp.path().join("repo");
19496        std::fs::create_dir_all(&repo).unwrap();
19497        init_repo(&repo);
19498        // Unique to this test, so holding its lease cannot collide with
19499        // another test sharing the same process-wide `home`.
19500        let cache_dir = tmp.path().join("target");
19501
19502        let mut config = Config::default();
19503        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19504        // `verify.gate` stays empty (the default). Any accidental cache
19505        // acquisition must report contention immediately, regardless of how
19506        // long unrelated Git subprocesses take under parallel test load.
19507        config.graph.timeout_verify = Some(2);
19508        config.graph.timeout_resource_wait = 0;
19509
19510        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19511        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19512            .expect("no io error acquiring directly")
19513        {
19514            crate::cache::AcquireOutcome::Acquired(g) => g,
19515            crate::cache::AcquireOutcome::Busy(b) => {
19516                panic!("expected the direct acquire to win the lease first: {b:?}")
19517            }
19518        };
19519
19520        let mut state = RunState::new(
19521            repo.clone(),
19522            "main".to_owned(),
19523            "deadbeef".to_owned(),
19524            "task".to_owned(),
19525            config,
19526        );
19527        state.candidates = vec![Candidate {
19528            index: 0,
19529            label: 'A',
19530            agent: "alpha".to_owned(),
19531            branch: "does-not-exist".to_owned(),
19532            worktree: repo.clone(),
19533            summary: String::new(),
19534            stat: String::new(),
19535            files: 0,
19536            commits: 0,
19537            empty: false,
19538            failed: None,
19539            verified_noop: None,
19540            duration_ms: 0,
19541            folded: false,
19542        }];
19543        state.tally = Some(Tally {
19544            first_choice: BTreeMap::from([('A', 1)]),
19545            borda: BTreeMap::new(),
19546            winner: 'A',
19547            rankings: 1,
19548            unanimous_initial: true,
19549            deliberated: false,
19550            changed_votes: 0,
19551            unanimous_final: true,
19552            tie_break: None,
19553            judges: 0,
19554            present: 0,
19555            quorum: 0,
19556            met_quorum: true,
19557            uncontested: Some("only candidate A produced a change".to_owned()),
19558        });
19559        state.reviews = vec![ReviewRound {
19560            round: 1,
19561            head: "deadbeef".to_owned(),
19562            verified_head: None,
19563            verified_at: None,
19564            reviews: Vec::new(),
19565            e2e: Vec::new(),
19566            fix: None,
19567            blocking: 0,
19568            answered: 0,
19569            expected: 0,
19570            clean: true,
19571            verify_retried: false,
19572            e2e_deferred: false,
19573            e2e_defer_reason: None,
19574            progressed: false,
19575            vote_split: false,
19576            reconsideration: Vec::new(),
19577            verdict: None,
19578        }];
19579
19580        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19581        state.reviews[0].e2e = vec![CommandOutcome {
19582            command: state.config.verify.e2e[0].clone(),
19583            code: Some(0),
19584            output_tail: String::new(),
19585            duration_ms: 0,
19586            resource_blocked: false,
19587        }];
19588        let mut runner = Runner {
19589            state,
19590            roles: ResolvedRoles {
19591                implementers: Vec::new(),
19592                judges: Vec::new(),
19593                reviewers: Vec::new(),
19594                fixer: None,
19595                conductor: conductor(),
19596                implementer_roster: Vec::new(),
19597                judge_roster: Vec::new(),
19598                reviewer_roster: Vec::new(),
19599            },
19600            sem: Arc::new(Semaphore::new(1)),
19601            pause: Pause::new(),
19602            interrupt: Pause::new(),
19603        };
19604
19605        runner.gate().await.expect("gate");
19606        assert!(runner.state.resource_block.is_none());
19607        assert!(
19608            !runner
19609                .state
19610                .events
19611                .iter()
19612                .any(|event| { event.message.contains("waiting for the shared build cache") }),
19613            "a gate with nothing to run must never request the held cache lease"
19614        );
19615        assert!(
19616            runner.state.gate_ran,
19617            "zero commands is still a real, immediate attempt"
19618        );
19619        assert!(runner.state.gate.is_empty());
19620        assert_ne!(
19621            runner.state.status,
19622            RunStatus::Blocked,
19623            "must not read as resource-blocked on a lease it never asked for"
19624        );
19625    }
19626
19627    /// A separate process carries a live worker's environment throughout all
19628    /// verification entry points; the parallel harness never changes its env.
19629    #[cfg(unix)]
19630    #[tokio::test]
19631    async fn verification_is_standalone_inside_a_live_worker() {
19632        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19633        if std::env::var_os(CHILD).is_none() {
19634            let home = tempfile::tempdir().unwrap();
19635            let generation = crate::worker::pin_current(home.path()).unwrap();
19636            let output = std::process::Command::new(std::env::current_exe().unwrap())
19637                .args([
19638                    "--exact",
19639                    "graph::tests::verification_is_standalone_inside_a_live_worker",
19640                    "--nocapture",
19641                ])
19642                .env(CHILD, "1")
19643                .env("MAGI_HOME", home.path())
19644                .env("MAGI_WORKER_HOME", home.path())
19645                .env("MAGI_WORKER_ID", "abcdef")
19646                .env("MAGI_WORKER_GENERATION", &generation.id)
19647                .env("MAGI_WORKER_EXE", &generation.executable)
19648                .env("MAGI_RUN", "verification-parent")
19649                .env("MAGI_NODE", "gate")
19650                .env("MAGI_SEAT", "worker-fixture")
19651                .env("MAGI_PROCESS_OWNER", "fixture-owner")
19652                .env("MAGI_NO_AUTOUPDATE", "0")
19653                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19654                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19655                .env(
19656                    "VERIFY_EXPECT_TOOLCHAIN",
19657                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19658                )
19659                .env(
19660                    "VERIFY_EXPECT_CACHE",
19661                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19662                )
19663                .output()
19664                .unwrap();
19665            assert!(
19666                output.status.success()
19667                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19668                "{}\n{}",
19669                String::from_utf8_lossy(&output.stdout),
19670                String::from_utf8_lossy(&output.stderr)
19671            );
19672            return;
19673        }
19674        let home = crate::run::home();
19675        let directory = home.join("workers/jobs/abcdef");
19676        std::fs::create_dir_all(&directory).unwrap();
19677        let job = crate::worker::Job {
19678            id: "abcdef".into(),
19679            sequence: 1,
19680            kind: "task".into(),
19681            key: "task:verification".into(),
19682            payload: serde_json::json!({}),
19683            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19684            version: Some(env!("CARGO_PKG_VERSION").into()),
19685            status: crate::worker::Status::Running,
19686            result: None,
19687            pid: Some(std::process::id()),
19688            created_at: 0,
19689            finished_at: None,
19690        };
19691        std::fs::write(
19692            directory.join("job.json"),
19693            serde_json::to_vec(&job).unwrap(),
19694        )
19695        .unwrap();
19696        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19697        let assert_worker = || {
19698            assert_eq!(
19699                crate::worker::current_job(&home).unwrap().unwrap().id,
19700                job.id
19701            );
19702            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19703            assert_eq!(
19704                crate::worker::helper_executable(&home).unwrap(),
19705                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19706            );
19707            assert!(
19708                crate::updater::Checker::new(&crate::config::Update {
19709                    mode: crate::config::UpdateMode::Notify,
19710                    interval: None,
19711                    auto_restart: false,
19712                })
19713                .is_none(),
19714                "the driver remains pinned, even while tests are standalone"
19715            );
19716        };
19717        assert_worker();
19718        let repo = home.join("repo");
19719        std::fs::create_dir_all(&repo).unwrap();
19720        init_repo(&repo);
19721        // Keep storage, seat/control context, containment and tool lookup.
19722        // No inherited values (or credentials) are printed on success.
19723        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
19724            test -z "${MAGI_WORKER_GENERATION+x}" &&
19725            test -z "${MAGI_WORKER_HOME+x}" &&
19726            test -z "${MAGI_WORKER_EXE+x}" &&
19727            test "$MAGI_HOME/repo" -ef . &&
19728            test "$MAGI_RUN" = verification-parent &&
19729            test "$MAGI_NODE" = gate &&
19730            test "$MAGI_SEAT" = worker-fixture &&
19731            test -n "$MAGI_PROCESS_OWNER" &&
19732            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
19733            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
19734            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
19735            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
19736        // Negative control: an ordinary child inherits the contamination.
19737        assert!(
19738            !std::process::Command::new("sh")
19739                .args(["-c", probe])
19740                .current_dir(&repo)
19741                .status()
19742                .unwrap()
19743                .success()
19744        );
19745        let unrelated_home = tempfile::tempdir().unwrap();
19746        assert!(
19747            crate::worker::current_job(unrelated_home.path()).is_err(),
19748            "an inherited worker id cannot be resolved in a unit fixture's home"
19749        );
19750        let mut runner = runner_at(RunStatus::Gating);
19751        runner.state.repo = repo.clone();
19752        runner.state.config.disk.min_free_bytes = 0;
19753        runner.state.config.coordination = crate::config::test_coordination();
19754        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
19755        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
19756        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
19757        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
19758        let winner = Candidate {
19759            index: 0,
19760            label: 'A',
19761            agent: "fixture".into(),
19762            branch: "main".into(),
19763            worktree: repo.clone(),
19764            summary: String::new(),
19765            stat: String::new(),
19766            files: 0,
19767            commits: 0,
19768            empty: false,
19769            failed: None,
19770            verified_noop: None,
19771            duration_ms: 0,
19772            folded: false,
19773        };
19774        let (e2e, retried, pids) = run_e2e_with_retry(
19775            &mut runner.state,
19776            &["sh".into(), "-c".into()],
19777            &[format!("{probe} && printf initial")],
19778            &repo,
19779            Duration::from_secs(10),
19780            "initial",
19781        )
19782        .await;
19783        assert!(!retried && pids.is_empty());
19784        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
19785        assert_eq!(e2e[0].output_tail, "initial");
19786        let gate = runner.run_gate(&winner).await.unwrap();
19787        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
19788        // An interrupted repair must recheck e2e from saved state. Landing
19789        // reuses that state and must run both pre-gate and gate again.
19790        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
19791            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
19792            passed: false,
19793            outcomes: Vec::new(),
19794        });
19795        runner.state.save().unwrap();
19796        let mut resumed = RunState::load(&runner.state.id).unwrap();
19797        let wait = crate::repo_lock::WaitPolicy::default();
19798        assert!(matches!(
19799            check_gate_repair_e2e(&mut resumed, &winner, &wait)
19800                .await
19801                .unwrap(),
19802            GateFix::Retry
19803        ));
19804        let saved = RunState::load(&resumed.id).unwrap();
19805        let repair = saved.gate_repair_e2e.unwrap();
19806        assert!(repair.passed);
19807        assert_eq!(repair.outcomes.len(), 1);
19808        assert!(repair.outcomes[0].ok());
19809        let landing = run_landing_gate(&mut resumed, &winner, &wait)
19810            .await
19811            .unwrap();
19812        assert_eq!(landing.len(), 1);
19813        assert!(landing[0].ok(), "{landing:?}");
19814        assert_eq!(
19815            std::fs::read_to_string(repo.join("phases")).unwrap(),
19816            "pre\ngate\ne2e\npre\ngate\n"
19817        );
19818        assert_worker();
19819    }
19820
19821    #[cfg(unix)]
19822    #[tokio::test]
19823    async fn verification_keeps_the_panic_and_the_full_output() {
19824        crate::run::pin_test_home();
19825        let tmp = tempfile::tempdir().unwrap();
19826        let mut config = Config::default();
19827        config.disk.min_free_bytes = 0;
19828        let mut state = RunState::new(
19829            tmp.path().to_owned(),
19830            "main".into(),
19831            "base".into(),
19832            "verification evidence".into(),
19833            config,
19834        );
19835        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
19836                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
19837                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
19838        let mut names = Vec::new();
19839        for _ in 0..2 {
19840            let (outcomes, _) = run_commands(
19841                &mut state,
19842                "gate",
19843                "gate",
19844                0,
19845                &["sh".into(), "-c".into()],
19846                &[script.into()],
19847                tmp.path(),
19848                Duration::from_secs(60),
19849            )
19850            .await;
19851            let tail = &outcomes[0].output_tail;
19852            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
19853            assert!(tail.contains("boom assertion"));
19854            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
19855            let line = tail.lines().next().unwrap();
19856            names.push(line.to_owned());
19857        }
19858        assert_ne!(names[0], names[1], "the second run must not overwrite");
19859        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
19860        assert!(full.len() > 500_000 && full.contains("boom assertion"));
19861        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
19862
19863        // Storage failures are evidence too; preserve the command's verdict.
19864        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
19865        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
19866        let (outcomes, _) = run_commands(
19867            &mut state,
19868            "gate",
19869            "gate",
19870            0,
19871            &["sh".into(), "-c".into()],
19872            &["echo failure-output; exit 7".into()],
19873            tmp.path(),
19874            Duration::from_secs(10),
19875        )
19876        .await;
19877        assert_eq!(outcomes[0].code, Some(7));
19878        assert!(state.events.iter().any(|e| {
19879            e.node == "gate" && e.message.contains("could not save the full gate output:")
19880        }));
19881    }
19882
19883    #[cfg(unix)]
19884    #[tokio::test]
19885    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
19886        crate::run::pin_test_home();
19887        let tmp = tempfile::tempdir().unwrap();
19888        let mut config = Config::default();
19889        config.disk.min_free_bytes = 0;
19890        let mut state = RunState::new(
19891            tmp.path().to_owned(),
19892            "main".into(),
19893            "base".into(),
19894            "verification evidence".into(),
19895            config,
19896        );
19897        let (outcomes, pids) = run_commands(
19898            &mut state,
19899            "gate",
19900            "gate",
19901            1,
19902            &["sh".into(), "-c".into()],
19903            &[
19904                "printf before-timeout; exec sleep 30".into(),
19905                "kill -TERM $$".into(),
19906                "exit 7".into(),
19907                "printf success".into(),
19908            ],
19909            tmp.path(),
19910            Duration::from_secs(1),
19911        )
19912        .await;
19913        wait_for_timed_out_children_to_die(&pids).await;
19914        assert_eq!(
19915            pids.len(),
19916            1,
19917            "unleased timeout retains its death-check PID"
19918        );
19919        assert_eq!(outcomes.len(), 4);
19920        assert_eq!(outcomes[0].code, None);
19921        assert!(outcomes[0].output_tail.contains("before-timeout"));
19922        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
19923        assert_eq!(outcomes[1].code, None);
19924        assert!(outcomes[1].output_tail.contains("command terminated:"));
19925        assert!(!outcomes[1].output_tail.contains("timed out"));
19926        assert_eq!(outcomes[2].code, Some(7));
19927        assert!(!outcomes[2].output_tail.contains("timed out"));
19928        assert_eq!(outcomes[3].code, Some(0));
19929        assert_eq!(outcomes[3].output_tail, "success");
19930        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
19931    }
19932
19933    /// The addendum's second gap: a `verify.gate` command running for real
19934    /// wall-clock time had nothing at all to show for it in `active` before
19935    /// `run_commands` learned to record it — a run could sit in `Gating` for
19936    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
19937    /// what was actually happening. Proven with a genuinely still-running
19938    /// command, not just a before/after check on the final state: a poller
19939    /// task reads the same `run.json` `gate()` is writing, the same way the
19940    /// phone or `magi show` would, while the shell command is still blocked
19941    /// on its own release marker.
19942    #[tokio::test]
19943    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
19944        crate::run::pin_test_home();
19945
19946        let tmp = tempfile::tempdir().expect("tempdir");
19947        let repo = tmp.path().join("repo");
19948        std::fs::create_dir_all(&repo).unwrap();
19949        init_repo(&repo);
19950
19951        let mut config = Config::default();
19952        config.disk.min_free_bytes = 0;
19953        config.verify.gate = vec![
19954            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
19955             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
19956                .to_owned(),
19957        ];
19958
19959        let mut state = RunState::new(
19960            repo.clone(),
19961            "main".to_owned(),
19962            "deadbeef".to_owned(),
19963            "task".to_owned(),
19964            config,
19965        );
19966        let run_id = state.id.clone();
19967        state.candidates = vec![Candidate {
19968            index: 0,
19969            label: 'A',
19970            agent: "alpha".to_owned(),
19971            branch: "does-not-exist".to_owned(),
19972            worktree: repo.clone(),
19973            summary: String::new(),
19974            stat: String::new(),
19975            files: 0,
19976            commits: 0,
19977            empty: false,
19978            failed: None,
19979            verified_noop: None,
19980            duration_ms: 0,
19981            folded: false,
19982        }];
19983        state.tally = Some(Tally {
19984            first_choice: BTreeMap::from([('A', 1)]),
19985            borda: BTreeMap::new(),
19986            winner: 'A',
19987            rankings: 1,
19988            unanimous_initial: true,
19989            deliberated: false,
19990            changed_votes: 0,
19991            unanimous_final: true,
19992            tie_break: None,
19993            judges: 0,
19994            present: 0,
19995            quorum: 0,
19996            met_quorum: true,
19997            uncontested: Some("only candidate A produced a change".to_owned()),
19998        });
19999        state.reviews = vec![ReviewRound {
20000            round: 1,
20001            head: "deadbeef".to_owned(),
20002            verified_head: None,
20003            verified_at: None,
20004            reviews: Vec::new(),
20005            e2e: Vec::new(),
20006            fix: None,
20007            blocking: 0,
20008            answered: 0,
20009            expected: 0,
20010            clean: true,
20011            verify_retried: false,
20012            e2e_deferred: false,
20013            e2e_defer_reason: None,
20014            progressed: false,
20015            vote_split: false,
20016            reconsideration: Vec::new(),
20017            verdict: None,
20018        }];
20019
20020        let mut runner = Runner {
20021            state,
20022            roles: ResolvedRoles {
20023                implementers: Vec::new(),
20024                judges: Vec::new(),
20025                reviewers: Vec::new(),
20026                fixer: None,
20027                conductor: conductor(),
20028                implementer_roster: Vec::new(),
20029                judge_roster: Vec::new(),
20030                reviewer_roster: Vec::new(),
20031            },
20032            sem: Arc::new(Semaphore::new(1)),
20033            pause: Pause::new(),
20034            interrupt: Pause::new(),
20035        };
20036
20037        let started_marker = repo.join("started.marker");
20038        let release_marker = repo.join("release.marker");
20039        let poller = tokio::spawn(async move {
20040            // Bounded so a regression that never records the task entry
20041            // fails this test in seconds instead of hanging the suite —
20042            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20043            // does_not_cut_it_short` uses for the same reason.
20044            for _ in 0..100 {
20045                if started_marker.exists()
20046                    && let Ok(s) = crate::run::RunState::load(&run_id)
20047                    && let Some(a) = s.active.get("gate")
20048                {
20049                    std::fs::write(&release_marker, b"go").expect("release marker");
20050                    return Some(a.clone());
20051                }
20052                tokio::time::sleep(Duration::from_millis(50)).await;
20053            }
20054            None
20055        });
20056
20057        runner.gate().await.expect("gate");
20058        let captured = poller.await.expect("poller task");
20059        let captured = captured.expect(
20060            "the poller never saw a `gate` task entry in run.json while the command was \
20061             still blocked on its own release marker",
20062        );
20063
20064        assert_eq!(captured.task.as_deref(), Some("gate"));
20065        assert_eq!(captured.node, "gate");
20066        assert_eq!(captured.index, Some(1));
20067        assert_eq!(captured.total, Some(1));
20068        assert!(
20069            captured
20070                .command
20071                .as_deref()
20072                .is_some_and(|c| c.contains("started.marker")),
20073            "{captured:?}"
20074        );
20075
20076        assert!(
20077            runner.state.active.is_empty(),
20078            "the entry must be cleared once the command actually finished: {:?}",
20079            runner.state.active
20080        );
20081        assert!(runner.state.gate_ran);
20082        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20083    }
20084
20085    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20086    /// record for `land`; one with only a Minor, or no reject, leaves none.
20087    #[tokio::test]
20088    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20089        use crate::verdict::{Finding, ReviewVote, Severity};
20090        crate::run::pin_test_home();
20091        let tmp = tempfile::tempdir().expect("tempdir");
20092        let repo = tmp.path().join("repo");
20093        std::fs::create_dir_all(&repo).unwrap();
20094        init_repo(&repo);
20095
20096        for (severity, vote, expect) in [
20097            (Severity::Major, ReviewVote::Reject, true),
20098            (Severity::Minor, ReviewVote::Reject, false),
20099            (Severity::Major, ReviewVote::Approve, false),
20100        ] {
20101            let mut round = review_round(false, 1, 1, 1, false, true);
20102            round.reviews = vec![ReviewRecord {
20103                reviewer: 1,
20104                agent: "alpha".to_owned(),
20105                summary: String::new(),
20106                findings: vec![Finding {
20107                    id: "R1-1-1".to_owned(),
20108                    severity,
20109                    file: None,
20110                    line: None,
20111                    title: "t".to_owned(),
20112                    detail: String::new(),
20113                }],
20114                vote: Some(vote),
20115                failed: None,
20116                duration_ms: 0,
20117                attempts: 0,
20118            }];
20119            let mut state = RunState::new(
20120                repo.clone(),
20121                "main".to_owned(),
20122                "deadbeef".to_owned(),
20123                "task".to_owned(),
20124                Config::default(),
20125            );
20126            state.reviews = vec![round];
20127            let mut runner = Runner {
20128                state,
20129                roles: ResolvedRoles {
20130                    implementers: Vec::new(),
20131                    judges: Vec::new(),
20132                    reviewers: Vec::new(),
20133                    fixer: None,
20134                    conductor: conductor(),
20135                    implementer_roster: Vec::new(),
20136                    judge_roster: Vec::new(),
20137                    reviewer_roster: Vec::new(),
20138                },
20139                sem: Arc::new(Semaphore::new(1)),
20140                pause: Pause::new(),
20141                interrupt: Pause::new(),
20142            };
20143            let shell = runner.state.config.shell();
20144            runner
20145                .stop_reviewing("round budget spent", &shell, &repo)
20146                .await
20147                .expect("stop_reviewing");
20148            assert_eq!(runner.state.status, RunStatus::Gating);
20149            assert_eq!(
20150                runner.state.contested_handoff.is_some(),
20151                expect,
20152                "{severity:?} + {vote:?}"
20153            );
20154        }
20155    }
20156
20157    /// The shape the incident this whole fix responds to actually had: the
20158    /// round budget spent, the last round's own e2e blocked on the shared
20159    /// build cache (held here by a live pid — this test process — exactly
20160    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20161    /// without forking a process). `stop_reviewing` must retry it — not
20162    /// silently leave the round looking untouched (the catch-up-only half of
20163    /// the bug), and not read the contention as a red `e2e` and block the
20164    /// run on it (the other half). Called directly, the same way
20165    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20166    /// above exercises `gate`, so this never needs a real cargo build to
20167    /// reach: the lease is never released, so `with_cache_lease` never gets
20168    /// past acquiring it into anything that would need a real workspace.
20169    #[tokio::test]
20170    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20171        crate::run::pin_test_home();
20172        let home = crate::run::home();
20173
20174        let tmp = tempfile::tempdir().expect("tempdir");
20175        let repo = tmp.path().join("repo");
20176        std::fs::create_dir_all(&repo).unwrap();
20177        init_repo(&repo);
20178        let head = crate::git::rev_parse(&repo, "HEAD")
20179            .await
20180            .expect("rev-parse");
20181        // Unique to this test, so holding its lease cannot collide with
20182        // another test sharing the same process-wide `home`.
20183        let cache_dir = tmp.path().join("target");
20184
20185        let mut config = Config::default();
20186        config.verify.e2e = vec![format!(
20187            "CARGO_TARGET_DIR='{}' test -f README.md",
20188            cache_dir.display()
20189        )];
20190        config.graph.review_rounds = 1;
20191        // Bounded so a regression that does start waiting fails the test in
20192        // seconds, not hangs it.
20193        config.graph.timeout_verify = Some(2);
20194        config.graph.timeout_resource_wait = 2;
20195
20196        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20197        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20198            .expect("no io error acquiring directly")
20199        {
20200            crate::cache::AcquireOutcome::Acquired(g) => g,
20201            crate::cache::AcquireOutcome::Busy(b) => {
20202                panic!("expected the direct acquire to win the lease first: {b:?}")
20203            }
20204        };
20205
20206        let mut state = RunState::new(
20207            repo.clone(),
20208            "main".to_owned(),
20209            head.clone(),
20210            "task".to_owned(),
20211            config,
20212        );
20213        state.candidates = vec![Candidate {
20214            index: 0,
20215            label: 'A',
20216            agent: "alpha".to_owned(),
20217            branch: "does-not-exist".to_owned(),
20218            worktree: repo.clone(),
20219            summary: String::new(),
20220            stat: String::new(),
20221            files: 0,
20222            commits: 0,
20223            empty: false,
20224            failed: None,
20225            verified_noop: None,
20226            duration_ms: 0,
20227            folded: false,
20228        }];
20229        state.tally = Some(Tally {
20230            first_choice: BTreeMap::from([('A', 1)]),
20231            borda: BTreeMap::new(),
20232            winner: 'A',
20233            rankings: 1,
20234            unanimous_initial: true,
20235            deliberated: false,
20236            changed_votes: 0,
20237            unanimous_final: true,
20238            tie_break: None,
20239            judges: 0,
20240            present: 0,
20241            quorum: 0,
20242            met_quorum: true,
20243            uncontested: Some("only candidate A produced a change".to_owned()),
20244        });
20245        // The round budget's last round, deferred: `needs_catchup_run`'s
20246        // other trigger. `stop_reviewing`'s retry machinery must treat this
20247        // exactly like a resource-blocked attempt once it actually runs.
20248        state.reviews = vec![ReviewRound {
20249            round: 1,
20250            head: head.clone(),
20251            verified_head: None,
20252            verified_at: None,
20253            reviews: Vec::new(),
20254            e2e: Vec::new(),
20255            fix: None,
20256            blocking: 1,
20257            answered: 1,
20258            expected: 1,
20259            clean: false,
20260            verify_retried: false,
20261            e2e_deferred: true,
20262            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20263            progressed: false,
20264            vote_split: false,
20265            reconsideration: Vec::new(),
20266            verdict: None,
20267        }];
20268
20269        let mut runner = Runner {
20270            state,
20271            roles: ResolvedRoles {
20272                implementers: Vec::new(),
20273                judges: Vec::new(),
20274                reviewers: Vec::new(),
20275                fixer: None,
20276                conductor: conductor(),
20277                implementer_roster: Vec::new(),
20278                judge_roster: Vec::new(),
20279                reviewer_roster: Vec::new(),
20280            },
20281            sem: Arc::new(Semaphore::new(1)),
20282            pause: Pause::new(),
20283            interrupt: Pause::new(),
20284        };
20285
20286        let shell = runner.state.config.shell();
20287        runner
20288            .stop_reviewing("round budget spent", &shell, &repo)
20289            .await
20290            .expect("stop_reviewing");
20291
20292        let last = runner.state.reviews.last().expect("round record");
20293        assert_eq!(
20294            last.e2e_status(),
20295            E2eStatus::ResourceBlocked,
20296            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20297             failed: {last:?}"
20298        );
20299        assert_eq!(
20300            last.verified_head.as_deref(),
20301            Some(head.as_str()),
20302            "which commit this attempt targeted is known even though nothing finished checking \
20303             it"
20304        );
20305        let first_attempt_at = last
20306            .verified_at
20307            .expect("when this attempt ran is known too");
20308        assert_ne!(
20309            runner.state.status,
20310            RunStatus::Blocked,
20311            "contention is evidence about the machine, not the patch — it must not settle the \
20312             run as blocked: {:?}",
20313            runner.state.status
20314        );
20315        assert!(
20316            !runner
20317                .state
20318                .events
20319                .iter()
20320                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20321            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20322            runner.state.events
20323        );
20324
20325        // The cache is still held: a later reentry must retry the same
20326        // round's verification again — not leave it looking exactly as
20327        // untouched as the first blocked attempt, which is indistinguishable
20328        // from never having tried again at all.
20329        runner
20330            .stop_reviewing("round budget spent", &shell, &repo)
20331            .await
20332            .expect("stop_reviewing retry");
20333        assert_eq!(
20334            runner.state.reviews.len(),
20335            1,
20336            "no new round was started: {:?}",
20337            runner.state.reviews
20338        );
20339        let last = runner.state.reviews.last().expect("round record");
20340        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20341        assert!(
20342            last.verified_at.expect("still known") > first_attempt_at,
20343            "a second reentry must be a fresh attempt, not a stale copy of the first"
20344        );
20345        assert_ne!(runner.state.status, RunStatus::Blocked);
20346
20347        held.release();
20348    }
20349
20350    #[cfg(unix)]
20351    #[tokio::test]
20352    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20353        ask_test_home();
20354        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20355            let tmp = tempfile::tempdir().unwrap();
20356            let repo = tmp.path().join("repo");
20357            std::fs::create_dir(&repo).unwrap();
20358            init_repo(&repo);
20359            let run_git = |args: &[&str]| {
20360                let out = std::process::Command::new("git")
20361                    .args(args)
20362                    .current_dir(&repo)
20363                    .output()
20364                    .unwrap();
20365                assert!(
20366                    out.status.success(),
20367                    "{args:?}: {}",
20368                    String::from_utf8_lossy(&out.stderr)
20369                );
20370            };
20371            let origin = tmp.path().join("origin.git");
20372            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20373            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20374            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20375            run_git(&["add", "spec.rs"]);
20376            run_git(&["commit", "-m", "initial spec"]);
20377            run_git(&["push", "origin", "main"]);
20378            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20379            run_git(&["checkout", "-b", "fix/review"]);
20380            std::fs::write(
20381                repo.join("check.rs"),
20382                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20383            )
20384            .unwrap();
20385            run_git(&["add", "check.rs"]);
20386            run_git(&["commit", "-m", "use spec"]);
20387            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20388            run_git(&["checkout", "main"]);
20389            std::fs::write(
20390                repo.join("spec.rs"),
20391                "struct AgentSpec { max_turn_secs: u64 }\n",
20392            )
20393            .unwrap();
20394            run_git(&["add", "spec.rs"]);
20395            run_git(&["commit", "-m", "extend spec"]);
20396            run_git(&["push", "origin", "main"]);
20397            run_git(&["checkout", "fix/review"]);
20398
20399            let mut runner = runner_at(RunStatus::Reviewing);
20400            runner.state.repo = repo.clone();
20401            runner.state.base_commit = base;
20402            runner.state.pr_head = Some("fix/review".into());
20403            runner.state.config.graph.review_rounds = max_rounds;
20404            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20405            runner.state.config.graph.sessions = false;
20406            runner.state.config.disk.min_free_bytes = 0;
20407            runner.state.config.verify.e2e =
20408                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20409            runner.state.candidates = vec![Candidate {
20410                index: 0,
20411                label: 'A',
20412                agent: "fixture".into(),
20413                branch: "fix/review".into(),
20414                worktree: repo.clone(),
20415                summary: String::new(),
20416                stat: String::new(),
20417                files: 1,
20418                commits: 1,
20419                empty: false,
20420                failed: None,
20421                verified_noop: None,
20422                duration_ms: 0,
20423                folded: false,
20424            }];
20425            runner.state.tally = Some(Tally {
20426                first_choice: BTreeMap::new(),
20427                borda: BTreeMap::new(),
20428                winner: 'A',
20429                rankings: 0,
20430                unanimous_initial: true,
20431                deliberated: false,
20432                changed_votes: 0,
20433                unanimous_final: true,
20434                tie_break: None,
20435                judges: 0,
20436                present: 0,
20437                quorum: 0,
20438                met_quorum: true,
20439                uncontested: Some("review-only".into()),
20440            });
20441            let mut last = review_round(false, 0, 1, 1, false, false);
20442            last.head = reviewed.clone();
20443            last.e2e_deferred = true;
20444            last.e2e[0].resource_blocked = true;
20445            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20446            if max_rounds == 1 {
20447                last.e2e.clear();
20448            }
20449            runner.state.reviews = vec![last];
20450            let calls = tmp.path().join("fix-calls");
20451            let mut fixer = spec("repair");
20452            fixer
20453                .env
20454                .insert("FIX_CALLS".into(), calls.display().to_string());
20455            fixer.command = vec!["sh".into(), "-c".into(), r#"
20456                echo fix >> "$FIX_CALLS"
20457                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20458                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20459                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20460                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20461            "#.into()];
20462            let mut reviewer = spec("reviewer");
20463            reviewer.command = vec![
20464                "sh".into(),
20465                "-c".into(),
20466                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20467                    .into(),
20468            ];
20469            runner.roles.fixer = Some(vec![fixer]);
20470            runner.roles.reviewers = vec![reviewer];
20471
20472            runner.sync_to_base().await.unwrap();
20473            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20474            assert_ne!(rebased, reviewed);
20475            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20476            if reentry {
20477                // Simulate a crash after catch-up evidence was saved but before
20478                // dispatching the fixer. No dedicated pending-repair field exists.
20479                let shell = runner.state.config.shell();
20480                runner
20481                    .stop_unresolved_review("deferred verification", &shell, &repo)
20482                    .await
20483                    .unwrap();
20484                assert!(runner.catchup_repairable());
20485                assert!(!calls.exists());
20486                runner.state = RunState::load(&runner.state.id).unwrap();
20487                assert!(runner.catchup_repairable());
20488            }
20489            runner.review_loop().await.unwrap();
20490            let first = &runner.state.reviews[0];
20491            assert_eq!(first.head, reviewed);
20492            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20493            assert!(
20494                first.verify_retried,
20495                "a reproduced build failure gets exactly one retry: {:?}",
20496                first.e2e
20497            );
20498            assert!(first.e2e[0].output_tail.contains("E0063"));
20499            assert!(!runner.catchup_repairable());
20500            if max_rounds == 2 {
20501                assert_eq!(
20502                    runner.state.status,
20503                    RunStatus::Gating,
20504                    "{}",
20505                    crate::report::run(&runner.state)
20506                );
20507                assert_eq!(runner.state.reviews.len(), 2);
20508                assert!(first.fix.as_ref().unwrap().committed);
20509                assert!(runner.state.reviews[1].clean);
20510                assert_eq!(
20511                    runner
20512                        .state
20513                        .reviews
20514                        .iter()
20515                        .filter(|r| r.fix.is_some())
20516                        .count(),
20517                    1
20518                );
20519                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20520                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20521            } else {
20522                assert_eq!(runner.state.status, RunStatus::Blocked);
20523                assert!(first.fix.is_none());
20524                assert!(!calls.exists());
20525                let recovery = crate::recovery::assess(
20526                    "prior",
20527                    Some(&runner.state),
20528                    &runner.landing_base(),
20529                    crate::recovery::Relation::Diverged,
20530                );
20531                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20532                let reasons = recovery.reasons.join("\n");
20533                assert!(
20534                    reasons.contains(&reviewed) && reasons.contains(&rebased),
20535                    "{reasons}"
20536                );
20537                assert!(
20538                    reasons.contains("base sync to")
20539                        && reasons.contains("additional review is required"),
20540                    "{reasons}"
20541                );
20542            }
20543        }
20544    }
20545
20546    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
20547    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
20548    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
20549    /// own top-of-function fast path (`review_conclusion`) correctly reads
20550    /// this shape as `None` rather than guessing `Blocked`, and the loop's
20551    /// own `for` range is empty once the round budget is spent, so
20552    /// `review_loop` must retry the check itself rather than silently doing
20553    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
20554    /// above exercises directly, but through `review_loop`'s own entry point
20555    /// this time, proving the wiring between the two rather than just the
20556    /// retry logic in isolation.
20557    #[tokio::test]
20558    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20559        crate::run::pin_test_home();
20560        let home = crate::run::home();
20561
20562        let tmp = tempfile::tempdir().expect("tempdir");
20563        let repo = tmp.path().join("repo");
20564        std::fs::create_dir_all(&repo).unwrap();
20565        init_repo(&repo);
20566        let head = crate::git::rev_parse(&repo, "HEAD")
20567            .await
20568            .expect("rev-parse");
20569        let cache_dir = tmp.path().join("target");
20570
20571        let mut config = Config::default();
20572        config.verify.e2e = vec![format!(
20573            "CARGO_TARGET_DIR='{}' test -f README.md",
20574            cache_dir.display()
20575        )];
20576        config.graph.review_rounds = 1;
20577        config.graph.timeout_verify = Some(2);
20578        config.graph.timeout_resource_wait = 2;
20579
20580        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20581        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20582            .expect("no io error acquiring directly")
20583        {
20584            crate::cache::AcquireOutcome::Acquired(g) => g,
20585            crate::cache::AcquireOutcome::Busy(b) => {
20586                panic!("expected the direct acquire to win the lease first: {b:?}")
20587            }
20588        };
20589
20590        let mut state = RunState::new(
20591            repo.clone(),
20592            "main".to_owned(),
20593            head.clone(),
20594            "task".to_owned(),
20595            config,
20596        );
20597        state.candidates = vec![Candidate {
20598            index: 0,
20599            label: 'A',
20600            agent: "alpha".to_owned(),
20601            branch: "does-not-exist".to_owned(),
20602            worktree: repo.clone(),
20603            summary: String::new(),
20604            stat: String::new(),
20605            files: 0,
20606            commits: 0,
20607            empty: false,
20608            failed: None,
20609            verified_noop: None,
20610            duration_ms: 0,
20611            folded: false,
20612        }];
20613        state.tally = Some(Tally {
20614            first_choice: BTreeMap::from([('A', 1)]),
20615            borda: BTreeMap::new(),
20616            winner: 'A',
20617            rankings: 1,
20618            unanimous_initial: true,
20619            deliberated: false,
20620            changed_votes: 0,
20621            unanimous_final: true,
20622            tie_break: None,
20623            judges: 0,
20624            present: 0,
20625            quorum: 0,
20626            met_quorum: true,
20627            uncontested: Some("only candidate A produced a change".to_owned()),
20628        });
20629        // The exact shape a prior process's `stop_reviewing` would have left
20630        // on disk: the round budget's last round, a real attempt already
20631        // made and already resource-blocked.
20632        state.reviews = vec![ReviewRound {
20633            round: 1,
20634            head: head.clone(),
20635            verified_head: Some(head.clone()),
20636            verified_at: Some(jiff::Timestamp::now()),
20637            reviews: Vec::new(),
20638            e2e: vec![CommandOutcome {
20639                command: format!(
20640                    "CARGO_TARGET_DIR='{}' test -f README.md",
20641                    cache_dir.display()
20642                ),
20643                code: None,
20644                output_tail: "waiting for the shared build cache".to_owned(),
20645                duration_ms: 0,
20646                resource_blocked: true,
20647            }],
20648            fix: None,
20649            blocking: 1,
20650            answered: 1,
20651            expected: 1,
20652            clean: false,
20653            verify_retried: false,
20654            e2e_deferred: false,
20655            e2e_defer_reason: None,
20656            progressed: false,
20657            vote_split: false,
20658            reconsideration: Vec::new(),
20659            verdict: None,
20660        }];
20661
20662        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20663        let mut runner = Runner {
20664            state,
20665            roles: ResolvedRoles {
20666                implementers: Vec::new(),
20667                judges: Vec::new(),
20668                reviewers: Vec::new(),
20669                fixer: None,
20670                conductor: conductor(),
20671                implementer_roster: Vec::new(),
20672                judge_roster: Vec::new(),
20673                reviewer_roster: Vec::new(),
20674            },
20675            sem: Arc::new(Semaphore::new(1)),
20676            pause: Pause::new(),
20677            interrupt: Pause::new(),
20678        };
20679
20680        // The lease is still held throughout, so this reentry's own retry is
20681        // also contended — proving `review_loop` actually tried again (not
20682        // that it happened to succeed) is what the timestamp comparison
20683        // below is for.
20684        runner.review_loop().await.expect("review_loop");
20685
20686        assert_eq!(
20687            runner.state.reviews.len(),
20688            1,
20689            "no new round was started on top of the unresolved one: {:?}",
20690            runner.state.reviews
20691        );
20692        let last = &runner.state.reviews[0];
20693        assert_eq!(
20694            last.e2e_status(),
20695            E2eStatus::ResourceBlocked,
20696            "still contended: {last:?}"
20697        );
20698        assert!(
20699            last.verified_at.expect("still known") > first_attempt_at,
20700            "review_loop must have actually retried the check, not left it exactly as found"
20701        );
20702        assert_ne!(
20703            runner.state.status,
20704            RunStatus::Blocked,
20705            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20706            runner.state.status
20707        );
20708
20709        held.release();
20710    }
20711
20712    #[tokio::test]
20713    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
20714        crate::run::pin_test_home();
20715        for status in [RunStatus::Blocked, RunStatus::Gating] {
20716            let mut runner = runner_at(status);
20717            runner.state.config.graph.land = true;
20718            runner.state.gate_ran = true;
20719            runner.state.merge = Some(MergeOutcome {
20720                mode: MergeMode::Pr,
20721                ok: false,
20722                detail: "the fixer produced no commit".into(),
20723                empty: false,
20724            });
20725            let url = "https://example.invalid/o/r/pull/1";
20726            runner.state.pr = Some(crate::run::PrRecord {
20727                url: url.into(),
20728                number: 1,
20729                state: "open".into(),
20730                checks: "red".into(),
20731                round: 1,
20732                rounds: 1,
20733                red_at_merge: Vec::new(),
20734            });
20735            runner.state.save().unwrap();
20736            // The nonexistent repository makes observation fail before a forge
20737            // command can run. We still must reach land, rather than silently
20738            // return from the graph with its old gate and merge records.
20739            runner.execute_graph().await.unwrap();
20740            assert_eq!(runner.state.status, RunStatus::Blocked);
20741            assert!(runner.state.land_sync_pending);
20742            assert!(runner.state.events.iter().any(|event|
20743                event.node == "land" && event.message == format!("watching {url}")));
20744            assert_eq!(
20745                runner.state.merge.as_ref().unwrap().detail,
20746                "the fixer produced no commit"
20747            );
20748        }
20749    }
20750
20751    #[tokio::test]
20752    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
20753        crate::run::pin_test_home();
20754        let tmp = tempfile::tempdir().expect("tempdir");
20755        let repo = tmp.path().join("repo");
20756        std::fs::create_dir_all(&repo).unwrap();
20757        init_repo(&repo);
20758
20759        let mut config = Config::default();
20760        config.merge.mode = MergeMode::Pr;
20761        config.graph.land = true;
20762        config.graph.land_approval = false;
20763
20764        let mut state = RunState::new(
20765            repo.clone(),
20766            "main".to_owned(),
20767            "deadbeef".to_owned(),
20768            "task".to_owned(),
20769            config,
20770        );
20771        state.candidates = vec![Candidate {
20772            index: 0,
20773            label: 'A',
20774            agent: "alpha".to_owned(),
20775            branch: "does-not-exist".to_owned(),
20776            worktree: repo.clone(),
20777            summary: String::new(),
20778            stat: String::new(),
20779            files: 0,
20780            commits: 0,
20781            empty: false,
20782            failed: None,
20783            verified_noop: None,
20784            duration_ms: 0,
20785            folded: false,
20786        }];
20787        state.tally = Some(Tally {
20788            first_choice: BTreeMap::from([('A', 1)]),
20789            borda: BTreeMap::new(),
20790            winner: 'A',
20791            rankings: 1,
20792            unanimous_initial: true,
20793            deliberated: false,
20794            changed_votes: 0,
20795            unanimous_final: true,
20796            tie_break: None,
20797            judges: 0,
20798            present: 0,
20799            quorum: 0,
20800            met_quorum: true,
20801            uncontested: Some("only candidate A produced a change".to_owned()),
20802        });
20803        state.reviews = vec![ReviewRound {
20804            round: 1,
20805            head: "deadbeef".to_owned(),
20806            verified_head: None,
20807            verified_at: None,
20808            reviews: Vec::new(),
20809            e2e: Vec::new(),
20810            fix: None,
20811            blocking: 0,
20812            answered: 0,
20813            expected: 0,
20814            clean: true,
20815            verify_retried: false,
20816            e2e_deferred: false,
20817            e2e_defer_reason: None,
20818            progressed: false,
20819            vote_split: false,
20820            reconsideration: Vec::new(),
20821            verdict: None,
20822        }];
20823        state.gate = vec![CommandOutcome {
20824            command: "test".to_owned(),
20825            code: Some(0),
20826            output_tail: String::new(),
20827            duration_ms: 0,
20828            resource_blocked: false,
20829        }];
20830        state.gate_ran = true;
20831        // A first pass through `merge` already pushed and opened this pull
20832        // request; `status` is `Landing` because a previous call into `land`
20833        // parked or was interrupted before it reached a terminal outcome.
20834        state.status = RunStatus::Landing;
20835        state.merge = Some(MergeOutcome {
20836            mode: MergeMode::Pr,
20837            ok: true,
20838            detail: "https://example.invalid/x/y/pull/1".to_owned(),
20839            empty: false,
20840        });
20841
20842        // The Landing-resume shortcut calls `run_land` directly rather than
20843        // through `merge`, which is exactly the call site that used to skip
20844        // `settle_questions` - see the fixture below.
20845        ask_test_home();
20846        // A real resume loads a persisted run; driver ownership is checked
20847        // against that record before the Landing shortcut is entered.
20848        state.save().expect("save landing run");
20849        let store = ask::Questions::open();
20850        let q = ask_open_question(&store, &state.id);
20851
20852        let mut runner = Runner {
20853            state,
20854            roles: ResolvedRoles {
20855                implementers: Vec::new(),
20856                judges: Vec::new(),
20857                reviewers: Vec::new(),
20858                fixer: None,
20859                conductor: conductor(),
20860                implementer_roster: Vec::new(),
20861                judge_roster: Vec::new(),
20862                reviewer_roster: Vec::new(),
20863            },
20864            sem: Arc::new(Semaphore::new(1)),
20865            pause: Pause::new(),
20866            interrupt: Pause::new(),
20867        };
20868
20869        // `execute`, not `merge` directly: the Landing-resume shortcut lives
20870        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
20871        // exactly because `review_loop` would otherwise clobber the marker
20872        // first.
20873        runner.execute().await.expect("execute");
20874
20875        assert_eq!(
20876            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20877            Some("https://example.invalid/x/y/pull/1"),
20878            "reentry must not push again or open a second pull request over the \
20879             one `land` is already watching"
20880        );
20881        assert_ne!(
20882            runner.state.status,
20883            RunStatus::Landing,
20884            "land could not actually reach the fake pull request, so it must \
20885             have given up rather than left the run silently parked forever"
20886        );
20887        // `land` could not reach the fake pull request, so it gave up into
20888        // `Blocked` - still resumable, so the question must not have been
20889        // swept just because this branch now also calls `settle_questions`.
20890        assert_eq!(runner.state.status, RunStatus::Blocked);
20891        assert!(
20892            store.get(&q.id).unwrap().status.open(),
20893            "Blocked is still alive; settle_questions must have been a no-op here"
20894        );
20895    }
20896
20897    fn state_with_round(round: ReviewRound) -> RunState {
20898        let mut s = RunState::new(
20899            PathBuf::from("/repo"),
20900            "main".to_owned(),
20901            "abc1234".to_owned(),
20902            "add retries".to_owned(),
20903            Config::default(),
20904        );
20905        s.reviews = vec![round];
20906        s
20907    }
20908
20909    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
20910        crate::verdict::Finding {
20911            id: id.to_owned(),
20912            severity,
20913            file: None,
20914            line: None,
20915            title: title.to_owned(),
20916            detail: String::new(),
20917        }
20918    }
20919
20920    #[test]
20921    fn pr_body_names_open_findings_and_declined_ones() {
20922        let round = ReviewRound {
20923            round: 2,
20924            head: "deadbee".to_owned(),
20925            verified_head: None,
20926            verified_at: None,
20927            reviews: vec![ReviewRecord {
20928                attempts: 0,
20929                reviewer: 1,
20930                agent: "alpha".to_owned(),
20931                summary: String::new(),
20932                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
20933                vote: None,
20934                failed: None,
20935                duration_ms: 0,
20936            }],
20937            e2e: vec![CommandOutcome {
20938                command: "cargo test".to_owned(),
20939                code: Some(0),
20940                output_tail: String::new(),
20941                duration_ms: 0,
20942                resource_blocked: false,
20943            }],
20944            verify_retried: false,
20945            e2e_deferred: false,
20946            e2e_defer_reason: None,
20947            fix: Some(FixRecord {
20948                agent: "alpha".to_owned(),
20949                addressed: Vec::new(),
20950                rejected: vec![crate::verdict::Rejection {
20951                    id: "R1-1-1".to_owned(),
20952                    why: "not reachable from any caller".to_owned(),
20953                }],
20954                notes: String::new(),
20955                committed: true,
20956                failed: None,
20957                duration_ms: 0,
20958                continuation: None,
20959            }),
20960            blocking: 0,
20961            answered: 1,
20962            expected: 1,
20963            clean: false,
20964            progressed: true,
20965            vote_split: false,
20966            reconsideration: Vec::new(),
20967            verdict: None,
20968        };
20969        let state = state_with_round(round);
20970        let body = pr_message(&state, 'A').body;
20971
20972        assert!(body.contains("add retries"), "the task must still be there");
20973        assert!(body.contains("R2-1-1"), "{body}");
20974        assert!(!body.contains("unused import"), "{body}");
20975        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
20976        assert!(
20977            !body.contains("not reachable from any caller"),
20978            "the reason it was declined: {body}"
20979        );
20980    }
20981
20982    #[test]
20983    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
20984        let round = ReviewRound {
20985            round: 1,
20986            head: "deadbee".to_owned(),
20987            verified_head: None,
20988            verified_at: None,
20989            reviews: vec![ReviewRecord {
20990                attempts: 0,
20991                reviewer: 1,
20992                agent: "alpha".to_owned(),
20993                summary: String::new(),
20994                findings: Vec::new(),
20995                vote: None,
20996                failed: None,
20997                duration_ms: 0,
20998            }],
20999            e2e: Vec::new(),
21000            verify_retried: false,
21001            e2e_deferred: false,
21002            e2e_defer_reason: None,
21003            fix: None,
21004            blocking: 0,
21005            answered: 1,
21006            expected: 1,
21007            clean: true,
21008            progressed: false,
21009            vote_split: false,
21010            reconsideration: Vec::new(),
21011            verdict: None,
21012        };
21013        let state = state_with_round(round);
21014        let body = pr_message(&state, 'A').body;
21015        assert!(!body.contains("Open review findings"), "{body}");
21016        assert!(!body.contains("Declined"), "{body}");
21017    }
21018
21019    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21020        let mut state = RunState::new(
21021            PathBuf::from("/repo"),
21022            "main".to_owned(),
21023            "abc1234".to_owned(),
21024            instruction.to_owned(),
21025            Config::default(),
21026        );
21027        // These fixtures test execution flow, independent of host disk capacity.
21028        state.config.disk.min_free_bytes = 0;
21029        state.candidates.push(Candidate {
21030            index: 0,
21031            label: 'A',
21032            agent: "alpha".to_owned(),
21033            branch: "magi/x/A".to_owned(),
21034            worktree: PathBuf::from("/wt"),
21035            summary: summary.to_owned(),
21036            stat: String::new(),
21037            files: 1,
21038            commits: 1,
21039            empty: false,
21040            failed: None,
21041            verified_noop: None,
21042            folded: false,
21043            duration_ms: 0,
21044        });
21045        state
21046    }
21047
21048    fn review_state(subjects: &[&str]) -> RunState {
21049        let mut state = state_with_summary(
21050            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21051            "",
21052        );
21053        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21054        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21055        state
21056    }
21057
21058    /// A branch rebased onto a main that moved past the recorded
21059    /// `base_commit` is titled from its own first commit, never main's.
21060    #[tokio::test]
21061    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21062        ask_test_home();
21063        let tmp = tempfile::tempdir().unwrap();
21064        let repo = tmp.path().join("repo");
21065        std::fs::create_dir_all(&repo).unwrap();
21066        init_repo(&repo);
21067        let origin = tmp.path().join("origin.git");
21068        let g = |dir: &Path, args: &[&str]| {
21069            let out = std::process::Command::new("git")
21070                .args(args)
21071                .current_dir(dir)
21072                .quiet()
21073                .output()
21074                .expect("spawn git");
21075            assert!(
21076                out.status.success(),
21077                "git {args:?}: {}",
21078                String::from_utf8_lossy(&out.stderr)
21079            );
21080        };
21081        g(
21082            tmp.path(),
21083            &[
21084                "clone",
21085                "--bare",
21086                "-q",
21087                repo.to_str().unwrap(),
21088                origin.to_str().unwrap(),
21089            ],
21090        );
21091        g(
21092            &repo,
21093            &["remote", "add", "origin", origin.to_str().unwrap()],
21094        );
21095        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21096
21097        // Main moves on; the branch is built on top of the new main.
21098        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21099        g(&repo, &["add", "-A"]);
21100        g(
21101            &repo,
21102            &["commit", "-q", "-m", "chore(deps): update a crate"],
21103        );
21104        g(&repo, &["push", "-q", "origin", "main"]);
21105        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21106        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21107        g(&repo, &["add", "-A"]);
21108        g(
21109            &repo,
21110            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21111        );
21112        g(&repo, &["checkout", "-q", "main"]);
21113
21114        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21115        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21116        // Without a readable tracking ref the recorded base's merge base is used.
21117        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21118        assert_eq!(fallback, c1);
21119
21120        let mut state = review_state(&[
21121            "chore(deps): update a crate",
21122            "fix(daemon): apply a chosen action",
21123        ]);
21124        state.repo = repo.clone();
21125        state.base_branch = "main".to_owned();
21126        state.base_commit = c1;
21127        refresh_reviewed_commits(&mut state, "feat/own").await;
21128        assert_eq!(
21129            state.reviewed_commits,
21130            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21131        );
21132        assert_eq!(
21133            review_title(&state).as_deref(),
21134            Some("fix(daemon): apply a chosen action")
21135        );
21136        assert_eq!(
21137            leaked_subjects(&state, "feat/own").await,
21138            Some(vec!["chore(deps): update a crate".to_owned()])
21139        );
21140        // A stale tracking ref is still used when the fetch fails, but the
21141        // leak list is withheld.
21142        state.config.merge.remote = "nowhere".to_owned();
21143        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21144    }
21145
21146    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21147    /// with `origin/main` one commit behind it, and a summarizing agent that
21148    /// prints whatever `reply` holds.
21149    struct SummaryFixture {
21150        _tmp: tempfile::TempDir,
21151        repo: PathBuf,
21152        reply: PathBuf,
21153        runner: Runner,
21154    }
21155
21156    fn summary_fixture() -> SummaryFixture {
21157        ask_test_home();
21158        let tmp = tempfile::tempdir().unwrap();
21159        let repo = tmp.path().join("repo");
21160        std::fs::create_dir_all(&repo).unwrap();
21161        init_repo(&repo);
21162        let origin = tmp.path().join("origin.git");
21163        let g = |args: &[&str]| {
21164            let out = std::process::Command::new("git")
21165                .args(args)
21166                .current_dir(&repo)
21167                .quiet()
21168                .output()
21169                .expect("spawn git");
21170            assert!(
21171                out.status.success(),
21172                "git {args:?}: {}",
21173                String::from_utf8_lossy(&out.stderr)
21174            );
21175        };
21176        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21177        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21178        g(&["push", "-q", "origin", "main"]);
21179        g(&["checkout", "-q", "-b", "feat/own"]);
21180        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21181        g(&["add", "-A"]);
21182        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21183        let reply = tmp.path().join("reply.txt");
21184        std::fs::write(
21185            &reply,
21186            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21187        )
21188        .unwrap();
21189        let mut runner = runner_at(RunStatus::Gating);
21190        let mut state = review_state(&["feat(own): add the own file"]);
21191        state.repo = repo.clone();
21192        state.base_branch = "main".to_owned();
21193        state.candidates[0].branch = "feat/own".to_owned();
21194        state.candidates[0].worktree = repo.clone();
21195        let mut author = spec("alpha");
21196        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21197        author
21198            .env
21199            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21200        state.config.agents = vec![author];
21201        runner.state = state;
21202        SummaryFixture {
21203            _tmp: tmp,
21204            repo,
21205            reply,
21206            runner,
21207        }
21208    }
21209
21210    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21211        let winner = f.runner.state.candidates[0].clone();
21212        f.runner
21213            .guarded_pr_message(&winner, None, true)
21214            .await
21215            .unwrap()
21216            .expect("an English description never parks")
21217    }
21218
21219    fn clean_round_with_minor_findings() -> ReviewRound {
21220        let mut round = passing_round(2, None);
21221        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21222        round.reviews = vec![ReviewRecord {
21223            attempts: 0,
21224            reviewer: 1,
21225            agent: "alpha".into(),
21226            summary: "内部レビュー".into(),
21227            findings: vec![
21228                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21229                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21230            ],
21231            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21232            failed: None,
21233            duration_ms: 0,
21234        }];
21235        round
21236    }
21237
21238    #[tokio::test]
21239    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21240        for review_only in [false, true] {
21241            let mut f = summary_fixture();
21242            if !review_only {
21243                f.runner.state.candidates[0].agent = "alpha".into();
21244            }
21245            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21246            assert!(f.runner.state.reviews[0].clean);
21247            assert!(f.runner.state.reviews[0].fix.is_none());
21248            assert!(f.runner.state.open_findings().is_empty());
21249            assert_eq!(
21250                crate::pr_outcome::required_finding_ids(&f.runner.state),
21251                ["R2-1-1", "R2-1-2"]
21252            );
21253            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21254            let evidence = crate::pr_outcome::summary_evidence(
21255                &f.runner.state,
21256                crate::pr_outcome::FinalHead::At(&head),
21257            );
21258            assert!(
21259                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21260            );
21261            // The ordinary reply omits both IDs: no cached description is adopted.
21262            let omitted = posted(&mut f).await;
21263            assert!(omitted.body.contains("No description"));
21264            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21265            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21266            assert!(f.runner.state.change_summary.is_none());
21267            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21268            let described = posted(&mut f).await;
21269            assert!(described.body.contains("improve documentation for callers"));
21270            assert!(described.body.contains("remaining edge case"));
21271            assert!(described.body.contains("## Open review findings"));
21272            assert!(!described.body.contains("未修正"));
21273        }
21274    }
21275
21276    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21277        let mut f = summary_fixture();
21278        if !review_only {
21279            f.runner.state.candidates[0].agent = "alpha".into();
21280        }
21281        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21282        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21283        fixed
21284            .fix
21285            .as_mut()
21286            .unwrap()
21287            .rejected
21288            .push(crate::verdict::Rejection {
21289                id: "R1-1-1".into(),
21290                why: "呼び出されないコードのため辞退".into(),
21291            });
21292        let mut clean = clean_round_with_minor_findings();
21293        clean.verified_head = Some(head.clone());
21294        f.runner.state.reviews = vec![fixed, clean];
21295        ok_gate(&mut f.runner.state, Some(&head));
21296        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21297        f.runner.state.config.agents.push(scripted("beta", reply));
21298        f.runner.state.config.roles.language_judge =
21299            Some(crate::config::AgentChoice::One("beta".into()));
21300        f
21301    }
21302
21303    #[tokio::test]
21304    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21305        for review_only in [false, true] {
21306            let mut f = rewrite_fixture_with_findings(review_only,
21307                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21308            let m = posted(&mut f).await;
21309            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21310            let expected =
21311                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21312            assert!(m.body.contains(&expected));
21313            assert!(m.body.contains("## Review rounds"));
21314            assert!(
21315                m.body
21316                    .contains("Gate (`verify.gate`): passed on the final commit")
21317            );
21318            assert!(
21319                m.body
21320                    .contains("Tests (`verify.e2e`): passed on the final commit")
21321            );
21322            let c = f.runner.state.change_summary.as_ref().unwrap();
21323            assert!(
21324                m.body
21325                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21326            );
21327            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21328            assert!(crate::pr_outcome::covers_required_findings(
21329                &f.runner.state,
21330                &m.body
21331            ));
21332            assert!(m.body.contains("documentation remains incomplete"));
21333            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21334            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21335            assert!(f.runner.state.github_text.is_none());
21336        }
21337    }
21338
21339    #[tokio::test]
21340    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21341        for review_only in [false, true] {
21342            for reply in [
21343                "TITLE: fix: public description\nA short English description.",
21344                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21345            ] {
21346                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21347                let winner = f.runner.state.candidates[0].clone();
21348                let posted = f
21349                    .runner
21350                    .guarded_pr_message(&winner, None, true)
21351                    .await
21352                    .unwrap();
21353                assert!(posted.is_none());
21354                assert!(f.runner.state.parked);
21355                let withheld = f.runner.state.github_text.as_ref().unwrap();
21356                assert!(
21357                    withheld.rewrite.is_none(),
21358                    "an incomplete rewrite must not be selectable"
21359                );
21360                let q = ask::Questions::open()
21361                    .get(withheld.question.as_ref().unwrap())
21362                    .unwrap();
21363                assert!(
21364                    !q.choices
21365                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21366                );
21367                assert!(
21368                    q.detail.contains("R1-1-1")
21369                        && q.detail.contains("R2-1-1")
21370                        && q.detail.contains("R2-1-2")
21371                );
21372            }
21373        }
21374    }
21375
21376    #[tokio::test]
21377    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21378        for review_only in [false, true] {
21379            let mut f = summary_fixture();
21380            if !review_only {
21381                f.runner.state.candidates[0].agent = "alpha".into();
21382            }
21383            f.runner.state.instruction = "内部タスクの説明".into();
21384            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21385            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21386            f.runner.state.reviews = vec![
21387                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21388                round_with_notes(2, Some("二回目の内部修正報告です。")),
21389                passing_round(3, Some(&head)),
21390            ];
21391            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21392            ok_gate(&mut f.runner.state, Some(&head));
21393            let captured = f.repo.join("public-prompt.txt");
21394            let author = &mut f.runner.state.config.agents[0];
21395            author.command = vec![
21396                "sh".into(),
21397                "-c".into(),
21398                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21399            ];
21400            author.env.insert(
21401                "PROMPT_FILE".into(),
21402                captured.to_string_lossy().into_owned(),
21403            );
21404            let m = posted(&mut f).await;
21405            assert!(m.body.contains("Adds `own.txt`"));
21406            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21407            assert!(m.body.contains("R1-1-1"));
21408            assert!(
21409                m.body
21410                    .contains("Gate (`verify.gate`): passed on the final commit")
21411            );
21412            assert!(
21413                m.body
21414                    .contains("Tests (`verify.e2e`): passed on the final commit")
21415            );
21416            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21417            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21418            let prompt = std::fs::read_to_string(captured).unwrap();
21419            assert!(prompt.contains(&format!(
21420                "git diff {} {head}",
21421                f.runner.state.change_summary.as_ref().unwrap().base
21422            )));
21423            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21424            assert_eq!(
21425                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21426                "二回目の内部修正報告です。"
21427            );
21428        }
21429    }
21430
21431    #[tokio::test]
21432    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21433        for review_only in [false, true] {
21434            let mut f = summary_fixture();
21435            if !review_only {
21436                f.runner.state.candidates[0].agent = "alpha".into();
21437            }
21438            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21439            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21440            std::fs::write(&f.reply, "no usable description").unwrap();
21441            let failed = posted(&mut f).await;
21442            assert!(
21443                failed
21444                    .body
21445                    .contains("No description of what this change does could be generated")
21446            );
21447            assert!(failed.body.is_ascii());
21448            assert!(f.runner.state.github_text.is_none());
21449            assert!(f.runner.state.change_summary.is_none());
21450            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21451            let evidence = crate::pr_outcome::summary_evidence(
21452                &f.runner.state,
21453                crate::pr_outcome::FinalHead::At(&head),
21454            );
21455            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21456            assert!(
21457                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21458                    .exists()
21459            );
21460            f.runner.state =
21461                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21462            std::fs::write(
21463                &f.reply,
21464                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21465            )
21466            .unwrap();
21467            let retried = posted(&mut f).await;
21468            assert!(retried.body.contains("Adds the committed own file."));
21469            assert_eq!(
21470                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21471                "日本語の修正報告"
21472            );
21473        }
21474    }
21475
21476    #[tokio::test]
21477    async fn same_head_verification_and_findings_invalidate_public_prose() {
21478        let mut f = summary_fixture();
21479        posted(&mut f).await;
21480        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21481        let original = f
21482            .runner
21483            .state
21484            .change_summary
21485            .as_ref()
21486            .unwrap()
21487            .evidence
21488            .clone();
21489        ok_gate(&mut f.runner.state, Some(&head));
21490        std::fs::write(
21491            &f.reply,
21492            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21493        )
21494        .unwrap();
21495        let verified = posted(&mut f).await;
21496        assert!(verified.body.contains("now verified"));
21497        assert_ne!(
21498            f.runner.state.change_summary.as_ref().unwrap().evidence,
21499            original
21500        );
21501        let mut round = round_with_notes(1, Some("内部報告"));
21502        round.reviews.push(ReviewRecord {
21503            attempts: 0,
21504            reviewer: 1,
21505            agent: "alpha".into(),
21506            summary: "内部レビュー".into(),
21507            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21508            vote: None,
21509            failed: None,
21510            duration_ms: 0,
21511        });
21512        round
21513            .fix
21514            .as_mut()
21515            .unwrap()
21516            .rejected
21517            .push(crate::verdict::Rejection {
21518                id: "R1-1-1".into(),
21519                why: "呼び出し元から到達できない".into(),
21520            });
21521        f.runner.state.reviews = vec![round];
21522        // A description omitting the required ID is unusable, even with a title.
21523        let omitted = posted(&mut f).await;
21524        assert!(omitted.body.contains("No description"));
21525        assert!(omitted.body.contains("R1-1-1"));
21526        assert!(!omitted.body.contains("到達"));
21527        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21528        let described = posted(&mut f).await;
21529        assert!(described.body.contains("unreachable from callers"));
21530        assert!(described.body.contains("R1-1-1"));
21531        assert!(!described.body.contains("到達"));
21532        let evidence = f
21533            .runner
21534            .state
21535            .change_summary
21536            .as_ref()
21537            .unwrap()
21538            .evidence
21539            .clone();
21540        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21541        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21542        assert!(posted(&mut f).await.body.contains("rationale updated"));
21543        assert_ne!(
21544            f.runner.state.change_summary.as_ref().unwrap().evidence,
21545            evidence
21546        );
21547        // Old serialized summaries have no evidence fingerprint and must regenerate.
21548        f.runner
21549            .state
21550            .change_summary
21551            .as_mut()
21552            .unwrap()
21553            .evidence
21554            .clear();
21555        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21556        assert!(posted(&mut f).await.body.contains("current specification"));
21557    }
21558
21559    #[tokio::test]
21560    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21561        let mut f = summary_fixture();
21562        let m = posted(&mut f).await;
21563        assert_eq!(m.title, "feat(own): add the own file");
21564        assert!(
21565            m.body
21566                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21567            "{}",
21568            m.body
21569        );
21570        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21571        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21572        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21573        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21574        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21575        let kept = f.runner.state.change_summary.as_ref().unwrap();
21576        assert_eq!(
21577            (kept.head.as_str(), kept.base.as_str()),
21578            (head.as_str(), base.as_str())
21579        );
21580        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21581    }
21582
21583    #[tokio::test]
21584    async fn review_only_summary_receives_the_configured_github_language() {
21585        let mut f = summary_fixture();
21586        f.runner.state.config.graph.github_language = "ja".into();
21587        let captured = f.repo.join("summary-prompt.txt");
21588        let author = &mut f.runner.state.config.agents[0];
21589        author.command = vec![
21590            "sh".into(),
21591            "-c".into(),
21592            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21593        ];
21594        author.env.insert(
21595            "PROMPT_FILE".into(),
21596            captured.to_string_lossy().into_owned(),
21597        );
21598        std::fs::write(
21599            &f.reply,
21600            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21601        )
21602        .unwrap();
21603        let m = posted(&mut f).await;
21604        let prompt = std::fs::read_to_string(captured).unwrap();
21605        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21606        assert!(!prompt.contains("in English"));
21607        assert_eq!(m.title, "feat(own): ファイルを追加");
21608        assert!(m.body.contains("own.txt を追加します。"));
21609    }
21610
21611    #[tokio::test]
21612    async fn review_fixes_come_after_the_whole_change_summary() {
21613        let mut f = summary_fixture();
21614        f.runner.state.reviews = vec![
21615            round_with_notes(1, Some("handled the PATH case")),
21616            round_with_notes(2, None),
21617        ];
21618        let m = posted(&mut f).await;
21619        let summary = m.body.find("## Summary").expect("summary");
21620        let fixes = m.body.find("## Review rounds").expect("review rounds");
21621        assert!(summary < fixes, "{}", m.body);
21622        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21623        assert!(m.body.contains("## Final verification"), "{}", m.body);
21624    }
21625
21626    #[tokio::test]
21627    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21628        let mut f = summary_fixture();
21629        let first = posted(&mut f).await;
21630        let first_head = f.runner.state.change_summary.clone().unwrap().head;
21631        // The same head reuses the record: a different reply is not read.
21632        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21633        let again = posted(&mut f).await;
21634        assert_eq!((again.title, again.body), (first.title, first.body));
21635
21636        let g = |args: &[&str]| {
21637            let out = std::process::Command::new("git")
21638                .args(args)
21639                .current_dir(&f.repo)
21640                .quiet()
21641                .output()
21642                .expect("spawn git");
21643            assert!(out.status.success(), "git {args:?}");
21644        };
21645        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21646        g(&["add", "-A"]);
21647        g(&["commit", "-q", "-m", "fix(own): handle more"]);
21648        let second = posted(&mut f).await;
21649        let kept = f.runner.state.change_summary.clone().unwrap();
21650        assert_ne!(kept.head, first_head);
21651        assert!(second.body.contains("Not read yet."), "{}", second.body);
21652        assert!(!second.body.contains("so the first summary is read"));
21653        assert_eq!(second.title, "feat(own): other");
21654    }
21655
21656    #[tokio::test]
21657    async fn an_unavailable_summary_is_stated_and_never_invented() {
21658        let mut f = summary_fixture();
21659        std::fs::write(&f.reply, "no title line here").unwrap();
21660        let m = posted(&mut f).await;
21661        assert!(
21662            m.body
21663                .contains("No description of what this change does could be generated"),
21664            "{}",
21665            m.body
21666        );
21667        assert!(m.body.contains("internal reports remain in local records"));
21668        assert!(f.runner.state.change_summary.is_none());
21669        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21670        // A preview asks nobody and claims nothing.
21671        let winner = f.runner.state.candidates[0].clone();
21672        let preview = f
21673            .runner
21674            .guarded_pr_message(&winner, None, false)
21675            .await
21676            .unwrap()
21677            .unwrap();
21678        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21679    }
21680
21681    #[test]
21682    fn pr_message_review_single_commit_uses_its_subject() {
21683        let state = review_state(&["feat(nats): per-role user"]);
21684        let m = pr_message(&state, 'A');
21685        assert_eq!(m.title, "feat(nats): per-role user");
21686        assert!(!m.body.contains("Review the work already"), "{}", m.body);
21687        assert!(m.body.contains("## Commits under review"), "{}", m.body);
21688    }
21689
21690    #[test]
21691    fn pr_message_review_multi_commit_takes_the_oldest() {
21692        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21693        let m = pr_message(&state, 'A');
21694        assert_eq!(m.title, "feat: the change");
21695        for s in ["feat: the change", "fix: typo", "fix: again"] {
21696            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21697        }
21698    }
21699
21700    #[test]
21701    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21702        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21703            let mut state = review_state(&[first, "fix: later fixup"]);
21704            state.candidates[0].branch = "機能/ログイン".to_owned();
21705            let m = pr_message(&state, 'A');
21706            assert!(
21707                m.title.starts_with("chore: land candidate A of run"),
21708                "{}",
21709                m.title
21710            );
21711        }
21712    }
21713
21714    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
21715        BranchFacts {
21716            commits: commits
21717                .iter()
21718                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
21719                .collect(),
21720            stat: stat.to_owned(),
21721        }
21722    }
21723
21724    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
21725        let mut r = review_round(true, 0, 1, 1, true, true);
21726        r.round = round;
21727        r.fix = notes.map(|n| FixRecord {
21728            agent: "fixer".to_owned(),
21729            addressed: Vec::new(),
21730            rejected: Vec::new(),
21731            notes: n.to_owned(),
21732            committed: true,
21733            failed: None,
21734            duration_ms: 0,
21735            continuation: None,
21736        });
21737        r
21738    }
21739
21740    #[test]
21741    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
21742        let state = review_state(&["ignored"]);
21743        let f = facts(
21744            &[
21745                (
21746                    "fix(login): resolve PATH on macOS",
21747                    "Login shells skip rc files.",
21748                ),
21749                ("fix: address review", ""),
21750            ],
21751            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
21752        );
21753        let m = pr_message_with(&state, 'A', Some(&f));
21754        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
21755        assert!(
21756            m.body
21757                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
21758            "{}",
21759            m.body
21760        );
21761        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
21762        assert!(m.body.contains("## Diff stat"), "{}", m.body);
21763        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
21764        for banned in [
21765            "Review the work already",
21766            "no task statement",
21767            "Original task",
21768        ] {
21769            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
21770        }
21771        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
21772    }
21773
21774    #[test]
21775    fn pr_message_review_truncates_a_huge_first_commit_body() {
21776        let state = review_state(&["ignored"]);
21777        let f = facts(
21778            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
21779            "s",
21780        );
21781        let m = pr_message_with(&state, 'A', Some(&f));
21782        assert!(m.body.len() < 30_000, "{}", m.body.len());
21783        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
21784        assert!(
21785            m.body.contains("- ... 1 more commit(s)"),
21786            "{}",
21787            m.body.len()
21788        );
21789        assert!(m.body.ends_with("magi:candidate-a\n"));
21790    }
21791
21792    #[test]
21793    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
21794        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
21795        assert_eq!(m.title, "feat: x");
21796        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
21797        assert!(!m.body.contains("Diff stat"), "{}", m.body);
21798        assert!(!m.body.contains("no task statement"), "{}", m.body);
21799    }
21800
21801    #[test]
21802    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
21803        let mut state = review_state(&["日本語の件名"]);
21804        state.candidates[0].branch = "fix/macos-login-path".to_owned();
21805        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
21806        state.candidates[0].branch = "機能/ログイン".to_owned();
21807        assert!(
21808            pr_message(&state, 'A')
21809                .title
21810                .starts_with("chore: land candidate A")
21811        );
21812    }
21813
21814    #[test]
21815    fn pr_message_review_fixes_survive_a_clean_final_round() {
21816        let mut state = review_state(&["feat: x"]);
21817        state.reviews = vec![
21818            round_with_notes(1, Some("handled the PATH case")),
21819            round_with_notes(2, None),
21820        ];
21821        let body = pr_message(&state, 'A').body;
21822        assert!(
21823            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
21824            "{body}"
21825        );
21826        assert!(body.contains("### Round 1"), "{body}");
21827
21828        state.reviews = vec![
21829            round_with_notes(1, Some("first fix")),
21830            round_with_notes(2, Some("")),
21831            round_with_notes(3, Some("second fix")),
21832            round_with_notes(4, None),
21833        ];
21834        let body = pr_message(&state, 'A').body;
21835        assert!(
21836            body.contains("### Round 1") && !body.contains("first fix"),
21837            "{body}"
21838        );
21839        assert!(
21840            body.contains("### Round 3") && !body.contains("second fix"),
21841            "{body}"
21842        );
21843        assert!(body.contains("### Round 2"), "{body}");
21844    }
21845
21846    const FINAL: &str = "abcdef1234567890";
21847
21848    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
21849        let mut r = round_with_notes(round, None);
21850        r.verified_head = verified.map(str::to_owned);
21851        r
21852    }
21853
21854    fn ok_gate(state: &mut RunState, head: Option<&str>) {
21855        state.gate = vec![CommandOutcome {
21856            command: "gate".to_owned(),
21857            code: Some(0),
21858            output_tail: String::new(),
21859            duration_ms: 0,
21860            resource_blocked: false,
21861        }];
21862        state.gate_ran = true;
21863        state.gate_head = head.map(str::to_owned);
21864    }
21865
21866    fn body_at(state: &RunState) -> String {
21867        pr_message_raw(
21868            state,
21869            'A',
21870            None,
21871            crate::pr_outcome::FinalHead::At(FINAL),
21872            ChangeNote::NotAsked,
21873        )
21874        .body
21875    }
21876
21877    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
21878
21879    #[test]
21880    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
21881        let mut state = review_state(&["feat: x"]);
21882        state.reviews = vec![
21883            round_with_notes(1, Some(SANDBOXED)),
21884            passing_round(2, Some(FINAL)),
21885        ];
21886        ok_gate(&mut state, Some(FINAL));
21887        let body = body_at(&state);
21888        assert!(!body.contains("Handled the PATH case."), "{body}");
21889        assert!(!body.contains("could not commit"), "{body}");
21890        assert!(!body.contains("could not run"), "{body}");
21891        assert!(
21892            body.contains("The fix is committed on the branch"),
21893            "{body}"
21894        );
21895        assert!(!body.contains("sandbox"), "{body}");
21896        assert!(
21897            body.contains("Tests (`verify.e2e`): passed on the final commit"),
21898            "{body}"
21899        );
21900        assert!(
21901            body.contains("Gate (`verify.gate`): passed on the final commit"),
21902            "{body}"
21903        );
21904        // The raw report is untouched in the record.
21905        assert!(
21906            state.reviews[0]
21907                .fix
21908                .as_ref()
21909                .unwrap()
21910                .notes
21911                .contains("could not commit")
21912        );
21913    }
21914
21915    #[test]
21916    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
21917        let mut state = review_state(&["feat: x"]);
21918        let mut r1 = round_with_notes(1, Some(SANDBOXED));
21919        r1.fix.as_mut().unwrap().committed = false;
21920        let mut r2 = passing_round(2, Some(FINAL));
21921        r2.e2e[0].code = Some(1);
21922        state.reviews = vec![r1, r2];
21923        let body = body_at(&state);
21924        assert!(!body.contains("could not commit"), "{body}");
21925        assert!(!body.contains("could not run"), "{body}");
21926        assert!(body.contains("This round produced no commit"), "{body}");
21927        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
21928        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
21929        assert!(!body.contains("passed"), "{body}");
21930    }
21931
21932    #[test]
21933    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
21934        let mut state = review_state(&["feat: x"]);
21935        state.reviews = vec![
21936            round_with_notes(1, Some(SANDBOXED)),
21937            passing_round(2, Some("0123456789abcdef")),
21938        ];
21939        ok_gate(&mut state, Some("0123456789abcdef"));
21940        let body = body_at(&state);
21941        assert!(!body.contains("HTTP tests could not run"), "{body}");
21942        // The commit claim is still settled by the structured record.
21943        assert!(!body.contains("could not commit"), "{body}");
21944        assert!(
21945            body.contains("passed on an earlier commit (`0123456`)"),
21946            "{body}"
21947        );
21948        assert!(!body.contains("passed on the final commit"), "{body}");
21949
21950        // A gate recorded before `gate_head` existed is "not recorded".
21951        ok_gate(&mut state, None);
21952        let body = body_at(&state);
21953        assert!(
21954            body.contains("passed on a commit that was not recorded"),
21955            "{body}"
21956        );
21957    }
21958
21959    #[test]
21960    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
21961        let state = state_with_summary(
21962            "add retries to the client",
21963            "TITLE: feat: retries\n\nDid it.",
21964        );
21965        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
21966        assert_eq!(m.title, "feat: retries");
21967        assert!(
21968            m.body.contains("<summary>Original task</summary>"),
21969            "{}",
21970            m.body
21971        );
21972        assert!(!m.body.contains("Commits under review"), "{}", m.body);
21973        assert!(
21974            m.body
21975                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
21976            "{}",
21977            m.body
21978        );
21979    }
21980
21981    #[test]
21982    fn pr_message_review_bounds_a_long_english_subject() {
21983        let long = format!("feat: {}", "word ".repeat(100));
21984        let m = pr_message(&review_state(&[&long]), 'A');
21985        assert!(m.title.starts_with("feat: word"), "{}", m.title);
21986        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
21987    }
21988
21989    #[test]
21990    fn pr_message_implementation_run_is_unchanged_by_review_support() {
21991        let state = state_with_summary("add retries\n\ndetails", "- did some things");
21992        let m = pr_message(&state, 'A');
21993        assert_eq!(m.title, "add retries");
21994        assert!(m.body.contains("<summary>Original task</summary>"));
21995        assert!(!m.body.contains("Commits under review"));
21996        assert_eq!(landing_subject_source(&state), state.instruction);
21997    }
21998
21999    #[test]
22000    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22001        let state = review_state(&["feat: the change", "fix: typo"]);
22002        let source = landing_subject_source(&state);
22003        assert_eq!(land::merge_subject("", &source), "feat: the change");
22004        assert_eq!(
22005            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22006            "feat: the change"
22007        );
22008        // An operator's rename still wins.
22009        assert_eq!(
22010            land::merge_subject("feat: renamed by hand", &source),
22011            "feat: renamed by hand"
22012        );
22013        let blank = review_state(&["日本語"]);
22014        assert!(
22015            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22016        );
22017    }
22018
22019    #[test]
22020    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22021        let state = review_state(&["feat: the change"]);
22022        let source = landing_subject_source(&state);
22023        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22024        assert_eq!(
22025            land::merge_subject(landing_title(&state, old), &source),
22026            "feat: the change"
22027        );
22028        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22029        let task = state_with_summary("add retries", "");
22030        assert_eq!(landing_title(&task, old), old);
22031    }
22032
22033    #[test]
22034    fn pr_message_describes_the_change_not_the_task() {
22035        let state = state_with_summary(
22036            "今回やってほしいこと: results projector を直す",
22037            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22038        );
22039        let m = pr_message(&state, 'A');
22040        assert_eq!(m.title, "fix(web): batch the runs list reads");
22041        assert!(
22042            m.body.starts_with("## Summary\n\n- reads run.json once"),
22043            "{}",
22044            m.body
22045        );
22046        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22047        let task_at = m.body.find("今回やってほしいこと").unwrap();
22048        let details_at = m.body.find("<details>").unwrap();
22049        assert!(
22050            details_at < task_at,
22051            "the task lives inside <details>: {}",
22052            m.body
22053        );
22054        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22055        assert!(m.body.contains("magi:candidate-a"));
22056    }
22057
22058    #[test]
22059    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22060        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22061        let m = pr_message(&state, 'A');
22062        assert_eq!(m.title, "add retries");
22063        assert!(
22064            m.body.contains("## Summary\n\n- did some things"),
22065            "{}",
22066            m.body
22067        );
22068
22069        let none = RunState::new(
22070            PathBuf::from("/repo"),
22071            "main".to_owned(),
22072            "abc1234".to_owned(),
22073            "add retries".to_owned(),
22074            Config::default(),
22075        );
22076        let m = pr_message(&none, 'A');
22077        assert_eq!(m.title, "add retries");
22078        assert!(!m.body.contains("## Summary"), "{}", m.body);
22079    }
22080
22081    #[test]
22082    fn pr_message_refuses_the_candidate_commit_subject() {
22083        for bad in [
22084            "TITLE: magi: candidate A (uncommitted work)",
22085            "TITLE: chore: stuff (uncommitted work)",
22086            "TITLE:   ",
22087        ] {
22088            let state = state_with_summary("add retries", bad);
22089            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22090        }
22091    }
22092
22093    #[test]
22094    fn pr_message_bounds_a_very_long_task_and_title() {
22095        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22096        let state = state_with_summary(&long, "- nothing");
22097        let m = pr_message(&state, 'A');
22098        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22099        assert!(!m.title.contains('\n'));
22100
22101        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22102        let m = pr_message(&state, 'A');
22103        assert!(m.title.starts_with("feat: "));
22104        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22105        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22106    }
22107
22108    fn long_title_of(instruction: &str) -> String {
22109        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22110    }
22111
22112    #[test]
22113    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22114        let first = "Make the landing path keep a readable title for long tasks";
22115        let line = format!(
22116            "{first}. {}",
22117            "And then keep going with more words ".repeat(20)
22118        );
22119        let t = long_title_of(&line);
22120        assert_eq!(t, first);
22121        assert!(!t.starts_with("chore: land"));
22122    }
22123
22124    #[test]
22125    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22126        let line = "word ".repeat(200);
22127        let t = long_title_of(&line);
22128        assert!(t.ends_with("word..."), "{t}");
22129        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22130    }
22131
22132    #[test]
22133    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22134        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22135            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22136        }
22137    }
22138
22139    #[test]
22140    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22141        let mut state = state_with_summary("再試行を追加する", "- nothing");
22142        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22143        state.config.graph.github_language = "ja".into();
22144        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22145    }
22146
22147    #[test]
22148    fn pr_message_title_limit_is_exact() {
22149        let at = "a".repeat(PR_TITLE_MAX);
22150        assert_eq!(long_title_of(&at), at);
22151        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22152        assert!(over.ends_with("..."), "{over}");
22153        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22154    }
22155
22156    #[test]
22157    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22158        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22159        let t = long_title_of(&line);
22160        assert!(t.ends_with("..."), "{t}");
22161        assert!(t.is_ascii(), "{t}");
22162    }
22163
22164    #[test]
22165    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22166        let line = format!(
22167            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22168            "plus more filler words ".repeat(20)
22169        );
22170        let t = long_title_of(&line);
22171        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22172    }
22173
22174    #[test]
22175    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22176        let line = format!("\n\n# {}", "title words ".repeat(40));
22177        let state = state_with_summary(&line, "- nothing");
22178        let m = pr_message(&state, 'A');
22179        assert!(m.title.starts_with("title words"), "{}", m.title);
22180        assert_eq!(
22181            land::merge_subject(&m.title, &landing_subject_source(&state)),
22182            m.title
22183        );
22184        // An operator's rename wins untouched.
22185        assert_eq!(
22186            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22187            "feat: renamed by hand"
22188        );
22189    }
22190
22191    #[tokio::test]
22192    async fn github_text_rewrite_is_bounded_and_falls_back() {
22193        crate::run::pin_test_home();
22194        for (reply, accepted) in [
22195            (
22196                "TITLE: fix: retries\nAdd retries for failed requests.",
22197                true,
22198            ),
22199            (
22200                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22201                false,
22202            ),
22203            (
22204                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22205                true,
22206            ),
22207            (
22208                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22209                true,
22210            ),
22211            (
22212                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22213                true,
22214            ),
22215            (
22216                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22217                false,
22218            ),
22219            ("TITLE: fix: retries\n", false),
22220        ] {
22221            let dir = tempfile::tempdir().unwrap();
22222            let mut runner = runner_at(RunStatus::Gating);
22223            runner.state = state_with_summary(
22224                "add retries",
22225                "TITLE: fix: retries\n日本語の説明を書きます。",
22226            );
22227            runner.state.repo = dir.path().to_owned();
22228            runner.state.candidates[0].worktree = dir.path().to_owned();
22229            let mut author = spec("alpha");
22230            author.command = vec![
22231                "sh".into(),
22232                "-c".into(),
22233                "printf '%s' \"$REWRITE_REPLY\"".into(),
22234            ];
22235            author.env.insert("REWRITE_REPLY".into(), reply.into());
22236            runner.state.config.agents = vec![author];
22237            seed_public_summary(&mut runner);
22238            let winner = runner.state.candidates[0].clone();
22239            let first = runner
22240                .guarded_pr_message(&winner, None, true)
22241                .await
22242                .unwrap();
22243            let message = if accepted {
22244                first.expect("a passing rewrite never asks")
22245            } else {
22246                // Withheld: the run parks on a question and posts nothing yet.
22247                assert!(first.is_none());
22248                assert!(runner.state.parked);
22249                let store = ask::Questions::open();
22250                let asked: Vec<_> = store
22251                    .list()
22252                    .into_iter()
22253                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22254                    .collect();
22255                assert_eq!(asked.len(), 1);
22256                assert!(!asked[0].detail.contains("secret"));
22257                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22258                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22259                assert_eq!(
22260                    asked[0]
22261                        .choices
22262                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22263                    rewrite.is_some()
22264                );
22265                // Silence: the question lapses and the neutral text is used,
22266                // without a second rewrite call or a second question.
22267                store
22268                    .update(&asked[0].id, |q| {
22269                        q.abandon("test");
22270                        Ok(())
22271                    })
22272                    .unwrap();
22273                let again = runner
22274                    .guarded_pr_message(&winner, None, true)
22275                    .await
22276                    .unwrap()
22277                    .expect("an abandoned question falls back");
22278                let n = store
22279                    .list()
22280                    .into_iter()
22281                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22282                    .count();
22283                assert_eq!(n, 1, "asked once per rejected text");
22284                again
22285            };
22286            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22287            assert!(!message.body.contains("secret"));
22288            assert!(!message.body.contains("prefix"));
22289            assert!(!message.title.contains("secret"));
22290            assert_eq!(
22291                message.body.contains("Add retries for failed requests."),
22292                accepted
22293            );
22294            assert_eq!(
22295                runner.state.seats["github-text-rewrite"].turns, 1,
22296                "only one rewrite invocation"
22297            );
22298            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22299            if accepted {
22300                assert!(runner.state.github_text.is_none());
22301                assert!(!runner.state.parked);
22302                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22303                assert!(message.body.contains("## Final verification"));
22304                assert!(message.body.contains("Evidence: final diff"));
22305                assert!(
22306                    message
22307                        .body
22308                        .contains(&format!("magi:run/{}", runner.state.id))
22309                );
22310            }
22311            if !accepted {
22312                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22313                assert!(
22314                    message
22315                        .body
22316                        .contains(&format!("magi:run/{}", runner.state.id))
22317                );
22318            }
22319        }
22320    }
22321
22322    fn scripted(id: &str, reply: &str) -> AgentSpec {
22323        let mut a = spec(id);
22324        a.command = vec![
22325            "sh".into(),
22326            "-c".into(),
22327            "printf '%s' \"$REWRITE_REPLY\"".into(),
22328        ];
22329        a.env.insert("REWRITE_REPLY".into(), reply.into());
22330        a
22331    }
22332
22333    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22334    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22335
22336    /// Gate tests inject generated public output, not internal reports.
22337    fn seed_public_summary(runner: &mut Runner) {
22338        let repo = &runner.state.repo;
22339        let g = |args: &[&str]| {
22340            let out = std::process::Command::new("git")
22341                .args(args)
22342                .current_dir(repo)
22343                .quiet()
22344                .output()
22345                .unwrap();
22346            assert!(out.status.success(), "git {args:?}");
22347            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22348        };
22349        g(&["init", "-q", "-b", "main"]);
22350        g(&[
22351            "-c",
22352            "user.name=Test",
22353            "-c",
22354            "user.email=test@example.invalid",
22355            "commit",
22356            "-q",
22357            "--allow-empty",
22358            "-m",
22359            "feat: fixture",
22360        ]);
22361        let head = g(&["rev-parse", "HEAD"]);
22362        g(&["update-ref", "refs/remotes/origin/main", &head]);
22363        let evidence = crate::pr_outcome::summary_evidence(
22364            &runner.state,
22365            crate::pr_outcome::FinalHead::At(&head),
22366        );
22367        let generated = &runner.state.candidates[0].summary;
22368        runner.state.change_summary = Some(crate::run::ChangeSummary {
22369            base: head.clone(),
22370            head,
22371            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22372            title: summary_title(generated).unwrap(),
22373            body: summary_without_title(generated),
22374        });
22375    }
22376
22377    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22378        let mut runner = runner_at(RunStatus::Gating);
22379        runner.state = state_with_summary("add retries", JA_SUMMARY);
22380        if review_only {
22381            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22382        }
22383        runner.state.repo = dir.to_owned();
22384        runner.state.candidates[0].worktree = dir.to_owned();
22385        runner.state.config.agents = agents;
22386        seed_public_summary(&mut runner);
22387        runner
22388    }
22389
22390    fn gate_questions(runner: &Runner) -> usize {
22391        ask::Questions::open()
22392            .list()
22393            .into_iter()
22394            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22395            .count()
22396    }
22397
22398    #[tokio::test]
22399    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22400        crate::run::pin_test_home();
22401        let dir = tempfile::tempdir().unwrap();
22402        let mut runner = rewrite_runner(
22403            dir.path(),
22404            true,
22405            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22406        );
22407        runner.state.config.roles.language_judge =
22408            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22409        let winner = runner.state.candidates[0].clone();
22410        let msg = runner
22411            .guarded_pr_message(&winner, None, true)
22412            .await
22413            .unwrap()
22414            .expect("an English rewrite posts without asking");
22415        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22416        assert!(msg.body.contains("<details>kept</details>"));
22417        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22418        assert_eq!(gate_questions(&runner), 0);
22419        assert!(runner.state.github_text.is_none());
22420        assert!(!runner.state.seats.contains_key("impl-A"));
22421        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22422    }
22423
22424    #[tokio::test]
22425    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22426        crate::run::pin_test_home();
22427        let dir = tempfile::tempdir().unwrap();
22428        let mut runner = rewrite_runner(
22429            dir.path(),
22430            false,
22431            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22432        );
22433        runner.state.config.roles.language_judge =
22434            Some(crate::config::AgentChoice::One("beta".into()));
22435        runner.state.seats.insert(
22436            "impl-A".into(),
22437            SeatState::new("impl-A", "alpha", runner.state.seed),
22438        );
22439        let winner = runner.state.candidates[0].clone();
22440        let msg = runner
22441            .guarded_pr_message(&winner, None, true)
22442            .await
22443            .unwrap()
22444            .expect("rewritten");
22445        assert!(msg.body.contains("<details>kept</details>"));
22446        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22447        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22448        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22449    }
22450
22451    #[tokio::test]
22452    async fn rewrite_reuses_a_resumable_implementer_session() {
22453        crate::run::pin_test_home();
22454        let dir = tempfile::tempdir().unwrap();
22455        let mut runner = rewrite_runner(
22456            dir.path(),
22457            false,
22458            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22459        );
22460        runner.state.config.roles.language_judge =
22461            Some(crate::config::AgentChoice::One("beta".into()));
22462        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22463        seat.turns = 1;
22464        runner.state.seats.insert("impl-A".into(), seat);
22465        let winner = runner.state.candidates[0].clone();
22466        runner
22467            .guarded_pr_message(&winner, None, true)
22468            .await
22469            .unwrap()
22470            .expect("rewritten");
22471        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22472        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22473    }
22474
22475    #[tokio::test]
22476    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22477        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22478        // session, was out of quota, and nothing else was asked.
22479        crate::run::pin_test_home();
22480        let dir = tempfile::tempdir().unwrap();
22481        let mut exhausted = spec("alpha");
22482        exhausted.command = vec!["false".into()];
22483        let mut runner = rewrite_runner(
22484            dir.path(),
22485            false,
22486            vec![exhausted, scripted("beta", EN_REWRITE)],
22487        );
22488        // The implementer's own agent is in the role too: it is not asked twice.
22489        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22490            "alpha".into(),
22491            "beta".into(),
22492        ]));
22493        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22494        seat.turns = 1;
22495        runner.state.seats.insert("impl-A".into(), seat);
22496        let winner = runner.state.candidates[0].clone();
22497        let msg = runner
22498            .guarded_pr_message(&winner, None, true)
22499            .await
22500            .unwrap()
22501            .expect("the role rewrites after the implementer fails");
22502        assert!(msg.body.contains("`src/net.rs`"));
22503        assert_eq!(
22504            runner.state.seats["impl-A"].turns, 2,
22505            "the implementer was tried once"
22506        );
22507        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22508        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22509    }
22510
22511    #[tokio::test]
22512    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22513        crate::run::pin_test_home();
22514        let dir = tempfile::tempdir().unwrap();
22515        let mut broken = spec("bad");
22516        broken.command = vec!["false".into()];
22517        let mut runner = rewrite_runner(
22518            dir.path(),
22519            true,
22520            vec![
22521                broken,
22522                scripted("noformat", "Add retries."),
22523                scripted("good", EN_REWRITE),
22524            ],
22525        );
22526        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22527            "bad".into(),
22528            "noformat".into(),
22529            "good".into(),
22530        ]));
22531        let winner = runner.state.candidates[0].clone();
22532        let msg = runner
22533            .guarded_pr_message(&winner, None, true)
22534            .await
22535            .unwrap()
22536            .expect("the third agent rewrites");
22537        assert!(msg.body.contains("`src/net.rs`"));
22538        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22539        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22540    }
22541
22542    #[tokio::test]
22543    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22544        crate::run::pin_test_home();
22545        // (agents, role): still Japanese; no TITLE; nothing resolvable.
22546        for (agents, role, saved) in [
22547            (
22548                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22549                Some("beta"),
22550                true,
22551            ),
22552            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22553            (Vec::new(), None, false),
22554        ] {
22555            let dir = tempfile::tempdir().unwrap();
22556            let mut runner = rewrite_runner(dir.path(), true, agents);
22557            runner.state.config.roles.language_judge =
22558                role.map(|r| crate::config::AgentChoice::One(r.into()));
22559            let winner = runner.state.candidates[0].clone();
22560            let first = runner
22561                .guarded_pr_message(&winner, None, true)
22562                .await
22563                .unwrap();
22564            assert!(first.is_none(), "withheld text parks the run");
22565            assert!(runner.state.parked);
22566            assert_eq!(gate_questions(&runner), 1);
22567            let g = runner.state.github_text.as_ref().unwrap();
22568            assert_eq!(g.rewrite.is_some(), saved);
22569            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22570        }
22571    }
22572
22573    #[tokio::test]
22574    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22575        crate::run::pin_test_home();
22576        let dir = tempfile::tempdir().unwrap();
22577        let mut runner = runner_at(RunStatus::Gating);
22578        runner.state =
22579            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22580        runner.state.repo = dir.path().to_owned();
22581        runner.state.candidates[0].worktree = dir.path().to_owned();
22582        let mut author = spec("alpha");
22583        author.command = vec![
22584            "sh".into(),
22585            "-c".into(),
22586            "printf '%s' \"$REWRITE_REPLY\"".into(),
22587        ];
22588        author.env.insert(
22589            "REWRITE_REPLY".into(),
22590            format!(
22591                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22592                "English sentence. ".repeat(600)
22593            ),
22594        );
22595        runner.state.config.agents = vec![author];
22596        seed_public_summary(&mut runner);
22597        let winner = runner.state.candidates[0].clone();
22598        assert!(
22599            runner
22600                .guarded_pr_message(&winner, None, true)
22601                .await
22602                .unwrap()
22603                .is_none()
22604        );
22605        let record = runner.state.github_text.clone().unwrap();
22606        let rewrite = record.rewrite.clone().unwrap();
22607        assert_eq!(rewrite.failed, ["body-language"]);
22608        assert!(
22609            rewrite
22610                .body
22611                .ends_with(&format!("magi:run/{}", runner.state.id))
22612        );
22613        let store = ask::Questions::open();
22614        let q = store.get(record.question.as_ref().unwrap()).unwrap();
22615        assert!(
22616            q.choices
22617                .contains(&crate::github_text::USE_REWRITE.to_owned())
22618        );
22619        for label in [
22620            "ORIGINAL",
22621            "FALLBACK",
22622            "REWRITE",
22623            "Rules still failed: body-language",
22624            "truncated",
22625        ] {
22626            assert!(q.detail.contains(label), "{label}");
22627        }
22628        let artifact = std::fs::read_to_string(crate::run::artifact_path(
22629            &runner.state,
22630            &crate::github_text::artifact_name(&record.fingerprint),
22631        ))
22632        .unwrap();
22633        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22634        // Restart from serialized state, then retry after a bad owner title.
22635        runner.state =
22636            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22637        store
22638            .update(&q.id, |q| {
22639                q.say("まだ日本語")?;
22640                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22641            })
22642            .unwrap();
22643        assert!(
22644            runner
22645                .guarded_pr_message(&winner, None, true)
22646                .await
22647                .unwrap()
22648                .is_none()
22649        );
22650        let retry = runner.state.github_text.as_ref().unwrap();
22651        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22652        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22653        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22654        assert!(
22655            q.posting_comparison
22656                .as_ref()
22657                .unwrap()
22658                .matches_detail(&q.detail)
22659        );
22660        store
22661            .update(&q.id, |q| {
22662                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22663            })
22664            .unwrap();
22665        let message = runner
22666            .guarded_pr_message(&winner, None, true)
22667            .await
22668            .unwrap()
22669            .unwrap();
22670        assert_eq!(message.title, rewrite.title);
22671        assert_eq!(message.body, rewrite.body);
22672        assert_eq!(
22673            runner.state.github_text.as_ref().unwrap().choice,
22674            Some(GithubTextChoice::Rewrite)
22675        );
22676        let again = runner
22677            .guarded_pr_message(&winner, None, true)
22678            .await
22679            .unwrap()
22680            .unwrap();
22681        assert_eq!(again.title, message.title);
22682        assert_eq!(again.body, message.body);
22683        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22684        assert!(
22685            !runner
22686                .state
22687                .events
22688                .iter()
22689                .any(|e| e.message.contains(&rewrite.title)
22690                    || e.message.contains("English sentence."))
22691        );
22692    }
22693
22694    #[tokio::test]
22695    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22696        crate::run::pin_test_home();
22697        let dir = tempfile::tempdir().unwrap();
22698        let mut runner = runner_at(RunStatus::Gating);
22699        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22700        runner.state.repo = dir.path().to_owned();
22701        runner.state.candidates[0].worktree = dir.path().to_owned();
22702        seed_public_summary(&mut runner);
22703        let winner = runner.state.candidates[0].clone();
22704        assert!(
22705            runner
22706                .guarded_pr_message(&winner, None, true)
22707                .await
22708                .unwrap()
22709                .is_none()
22710        );
22711        let record = runner.state.github_text.as_ref().unwrap();
22712        assert!(record.rewrite.is_none());
22713        // Records written before the additive field deserialize with no rewrite.
22714        let mut old = serde_json::to_value(record).unwrap();
22715        old.as_object_mut().unwrap().remove("rewrite");
22716        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
22717        assert!(old.rewrite.is_none());
22718        let id = record.question.clone().unwrap();
22719        ask::Questions::open()
22720            .update(&id, |q| {
22721                q.choices.push(crate::github_text::USE_REWRITE.into());
22722                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22723            })
22724            .unwrap();
22725        let message = runner
22726            .guarded_pr_message(&winner, None, true)
22727            .await
22728            .unwrap()
22729            .unwrap();
22730        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22731        assert_eq!(
22732            runner.state.github_text.as_ref().unwrap().choice,
22733            Some(GithubTextChoice::Fallback)
22734        );
22735    }
22736
22737    #[tokio::test]
22738    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
22739        crate::run::pin_test_home();
22740        let dir = tempfile::tempdir().unwrap();
22741        let mut runner = runner_at(RunStatus::Gating);
22742        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
22743        runner.state.repo = dir.path().to_owned();
22744        runner.state.candidates[0].worktree = dir.path().to_owned();
22745        seed_public_summary(&mut runner);
22746        let winner = runner.state.candidates[0].clone();
22747        assert!(
22748            runner
22749                .guarded_pr_message(&winner, None, true)
22750                .await
22751                .unwrap()
22752                .is_none()
22753        );
22754        let store = ask::Questions::open();
22755        let find = |run: &str| {
22756            store
22757                .list()
22758                .into_iter()
22759                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
22760                .unwrap()
22761        };
22762        let run = runner.state.id.clone();
22763        let q = find(&run);
22764        let fp = runner
22765            .state
22766            .github_text
22767            .as_ref()
22768            .unwrap()
22769            .fingerprint
22770            .clone();
22771        let name = crate::github_text::artifact_name(&fp);
22772        let path = crate::run::artifact_path(&runner.state, &name);
22773        assert!(q.detail.contains(&path.display().to_string()));
22774        assert!(q.detail.contains("title-language: title"));
22775        let saved = std::fs::read_to_string(&path).expect("artifact written");
22776        assert!(saved.contains("日本語のタイトル"));
22777        assert!(q.cwd.is_none());
22778        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
22779        // A bad replacement is refused and asked about once more.
22780        let say = |id: &str, text: &str, choice: &str| {
22781            store
22782                .update(id, |q| {
22783                    q.thread.push(ask::Turn {
22784                        who: ask::Who::Operator,
22785                        body: text.to_owned(),
22786                        at: jiff::Timestamp::now(),
22787                        note: None,
22788                    });
22789                    q.answer(ask::Answer::Choice(choice.to_owned()))
22790                })
22791                .unwrap();
22792        };
22793        say(&q.id, "まだ日本語", "use my text");
22794        assert!(
22795            runner
22796                .guarded_pr_message(&winner, None, true)
22797                .await
22798                .unwrap()
22799                .is_none()
22800        );
22801        let second = find(&run);
22802        assert_ne!(second.id, q.id);
22803        assert!(second.detail.contains("did not pass"));
22804        // The retry still compares the original text; the rejected
22805        // replacement is named by category only.
22806        assert!(second.detail.contains("日本語のタイトル"));
22807        assert!(!second.detail.contains("まだ日本語"));
22808        assert!(second.detail.contains(&path.display().to_string()));
22809        say(&second.id, "fix: retry failed requests", "use my text");
22810        let message = runner
22811            .guarded_pr_message(&winner, None, true)
22812            .await
22813            .unwrap()
22814            .expect("a vetted replacement is posted");
22815        assert_eq!(message.title, "fix: retry failed requests");
22816        assert_eq!(
22817            runner
22818                .state
22819                .github_text
22820                .as_ref()
22821                .unwrap()
22822                .chosen_title
22823                .as_deref(),
22824            Some("fix: retry failed requests")
22825        );
22826        assert!(runner.state.github_text.as_ref().unwrap().resolved);
22827        // Resumed again: the saved decision is applied, nothing is re-asked.
22828        let again = runner
22829            .guarded_pr_message(&winner, None, true)
22830            .await
22831            .unwrap()
22832            .unwrap();
22833        assert_eq!(again.title, message.title);
22834        assert_eq!(
22835            store
22836                .list()
22837                .into_iter()
22838                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
22839                .count(),
22840            2
22841        );
22842    }
22843
22844    #[test]
22845    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
22846        // What magi itself writes stays English under any configured language,
22847        // so a future localisation of these headings fails here. (The agents'
22848        // own text is also checked by the posting gate.)
22849        let mut state = state_with_summary(
22850            "add retries",
22851            "TITLE: fix(web): batch reads\n- reads run.json once",
22852        );
22853        state.config.graph.language = "ja".to_owned();
22854        let m = pr_message(&state, 'A');
22855        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
22856        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
22857
22858        // The task is the operator's own text: it goes in untouched, and the
22859        // fallback title (no summary) may be in its language too.
22860        let task = "今回やってほしいこと: results projector を直す";
22861        let mut state = state_with_summary(task, "- no title line");
22862        state.config.graph.language = "ja".to_owned();
22863        let m = pr_message(&state, 'A');
22864        assert_eq!(
22865            m.title,
22866            format!("chore: land candidate A of run {}", state.id)
22867        );
22868        assert!(
22869            m.body.contains(&format!(
22870                "<summary>Original task</summary>\n\n{task}\n\n</details>"
22871            )),
22872            "{}",
22873            m.body
22874        );
22875    }
22876
22877    #[test]
22878    fn pr_message_scrubs_home_paths_and_addresses() {
22879        let state = state_with_summary(
22880            "fix it in /Users/someone/src/x",
22881            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
22882        );
22883        let m = pr_message(&state, 'A');
22884        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
22885            assert!(!m.body.contains(leak), "{}", m.body);
22886        }
22887        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
22888    }
22889
22890    #[test]
22891    fn pr_message_survives_a_task_that_closes_details() {
22892        let state = state_with_summary("a </details> b", "TITLE: fix: x");
22893        let m = pr_message(&state, 'A');
22894        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
22895    }
22896
22897    #[test]
22898    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
22899        let cmd = manual_merge_command(
22900            MergeStyle::Squash,
22901            Path::new("/repo"),
22902            "b",
22903            "fix: \"quoted\" $(x) `y`\n\nbody",
22904        );
22905        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
22906    }
22907
22908    #[test]
22909    fn manual_merge_command_matches_the_configured_style() {
22910        let repo = Path::new("/repo");
22911        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
22912
22913        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
22914        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
22915
22916        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
22917        assert_eq!(
22918            squash,
22919            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
22920             \"Merge magi run 0832 (candidate A)\""
22921        );
22922
22923        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
22924        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
22925    }
22926
22927    #[test]
22928    fn a_nudge_gets_a_quarter_of_the_budget() {
22929        // The judge and implement budgets magi ships with.
22930        assert_eq!(retry_budget(secs(1200), true), secs(300));
22931        assert_eq!(retry_budget(secs(3600), true), secs(900));
22932    }
22933
22934    #[test]
22935    fn a_resent_prompt_keeps_the_whole_budget() {
22936        // The seat kept no context, so the retry is the original job again and
22937        // shortening it would only guarantee a second failure.
22938        assert_eq!(retry_budget(secs(1200), false), secs(1200));
22939        assert_eq!(retry_budget(secs(60), false), secs(60));
22940    }
22941
22942    #[test]
22943    fn the_floor_never_exceeds_the_original_budget() {
22944        // A short configured timeout must not be *raised* by the floor: the
22945        // operator asked for a bound, and a retry may not outlast the attempt
22946        // it is retrying.
22947        assert_eq!(retry_budget(secs(60), true), secs(60));
22948        assert_eq!(retry_budget(secs(480), true), secs(120));
22949        assert_eq!(retry_budget(secs(0), true), secs(0));
22950    }
22951
22952    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
22953        agent::CommandEvidence {
22954            id: "item1".to_owned(),
22955            description: "cargo test".to_owned(),
22956            exit_code,
22957            result_summary: String::new(),
22958            source: "codex".to_owned(),
22959        }
22960    }
22961
22962    #[test]
22963    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
22964        // No evidence is not the same fact as unconfirmed evidence: a
22965        // backend with no adapter, or a reply that ran no commands at all,
22966        // must not be misread as carrying a dangling job.
22967        assert!(!has_unconfirmed_command(&[]));
22968    }
22969
22970    #[test]
22971    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
22972        // Deliberately not a check on the exit code's *value*: a fixer
22973        // legitimately runs something that fails mid-iteration before it
22974        // succeeds, and that must never by itself reopen a valid report.
22975        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
22976        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
22977        assert!(!has_unconfirmed_command(&[
22978            evidence(Some(0)),
22979            evidence(Some(101))
22980        ]));
22981    }
22982
22983    #[test]
22984    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
22985        assert!(has_unconfirmed_command(&[
22986            evidence(Some(0)),
22987            evidence(None)
22988        ]));
22989    }
22990
22991    #[test]
22992    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
22993        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
22994        assert_eq!(
22995            verified_noop_claim(true, &[], text).as_deref(),
22996            Some("already fixed by b32cfc4, on main.")
22997        );
22998    }
22999
23000    #[test]
23001    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23002        // A timeout or a bad exit code reads as the ordinary loss it is,
23003        // whatever the reply's own prose claims.
23004        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23005        assert!(verified_noop_claim(false, &[], text).is_none());
23006    }
23007
23008    #[test]
23009    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23010        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23011        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23012        // A confirmed command alongside the marker is fine.
23013        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23014    }
23015
23016    #[test]
23017    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23018        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23019    }
23020
23021    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23022    /// pair, in order, labelled A, B, C, ...
23023    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23024        runner.state.candidates = shape
23025            .iter()
23026            .enumerate()
23027            .map(|(i, &(empty, verified))| Candidate {
23028                index: i,
23029                label: (b'A' + i as u8) as char,
23030                agent: "sonnet".to_owned(),
23031                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23032                worktree: PathBuf::from(format!("/wt/{i}")),
23033                summary: String::new(),
23034                stat: String::new(),
23035                files: 0,
23036                commits: 0,
23037                empty,
23038                failed: None,
23039                verified_noop: verified.map(str::to_owned),
23040                duration_ms: 0,
23041                folded: false,
23042            })
23043            .collect();
23044    }
23045
23046    #[test]
23047    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23048        ask_test_home();
23049        let mut runner = runner_at(RunStatus::Implementing);
23050        set_candidates(
23051            &mut runner,
23052            &[
23053                (true, Some("already on main at b32cfc4")),
23054                (true, Some("same fix, see the existing test")),
23055            ],
23056        );
23057
23058        runner
23059            .after_implement()
23060            .expect("a verified no-op is not an error");
23061
23062        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23063    }
23064
23065    #[test]
23066    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23067        ask_test_home();
23068        let mut runner = runner_at(RunStatus::Implementing);
23069        // Candidate A declares a verified no-op; candidate B simply wrote
23070        // nothing and said nothing about why. One candidate's claim is not
23071        // the whole run's agreement.
23072        set_candidates(
23073            &mut runner,
23074            &[(true, Some("already on main at b32cfc4")), (true, None)],
23075        );
23076
23077        let err = runner
23078            .after_implement()
23079            .expect_err("an unverified empty candidate must still fail the run");
23080
23081        assert!(
23082            err.to_string().contains("no candidate produced a change"),
23083            "{err}"
23084        );
23085        assert_eq!(runner.state.status, RunStatus::Failed);
23086    }
23087
23088    #[test]
23089    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23090        ask_test_home();
23091        let mut runner = runner_at(RunStatus::Implementing);
23092        set_candidates(&mut runner, &[(true, None), (true, None)]);
23093
23094        let err = runner
23095            .after_implement()
23096            .expect_err("no candidate declared anything; this is an ordinary failure");
23097
23098        assert!(
23099            err.to_string().contains("no candidate produced a change"),
23100            "{err}"
23101        );
23102        assert_eq!(runner.state.status, RunStatus::Failed);
23103    }
23104}
23105
23106/// A graft rollback whose restore could not be confirmed (see
23107/// [`Runner::graft`]).
23108#[derive(Debug)]
23109struct GraftRestoreFailed(String);
23110
23111impl std::fmt::Display for GraftRestoreFailed {
23112    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23113        write!(
23114            f,
23115            "could not restore the winner to its pre-graft commit: {}",
23116            self.0
23117        )
23118    }
23119}
23120
23121impl std::error::Error for GraftRestoreFailed {}
23122
23123/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23124/// not the agent: they never draw on an agent's own timeout, and running out of
23125/// them never reads as an agent failure.
23126#[cfg(test)]
23127mod resource_wait_tests {
23128    use super::*;
23129    use std::collections::BTreeMap;
23130
23131    fn sh(id: &str, script: &str) -> AgentSpec {
23132        AgentSpec {
23133            id: id.to_owned(),
23134            kind: crate::config::AgentKind::Command,
23135            model: None,
23136            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23137            extra_args: Vec::new(),
23138            env: BTreeMap::new(),
23139            prompt_delivery: None,
23140            max_turn_secs: None,
23141        }
23142    }
23143
23144    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23145        SeatJob {
23146            seat: SeatState::new("impl-A", &spec.id, 1),
23147            spec,
23148            cwd: dir.to_path_buf(),
23149            prompt: "do it".to_owned(),
23150            timeout,
23151            allow_write: true,
23152            sessions: false,
23153            artifacts: dir.join("art"),
23154            stem: "impl-A".to_owned(),
23155            handover: None,
23156        }
23157    }
23158
23159    fn state(resource_wait_secs: u64) -> RunState {
23160        let mut config = Config::default();
23161        config.graph.timeout_resource_wait = resource_wait_secs;
23162        RunState::new(
23163            PathBuf::from("/repo"),
23164            "main".to_owned(),
23165            "abc1234".to_owned(),
23166            "task".to_owned(),
23167            config,
23168        )
23169    }
23170
23171    fn hold(cache: &Path) -> crate::cache::Guard {
23172        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23173        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23174            crate::cache::AcquireOutcome::Acquired(g) => g,
23175            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23176        }
23177    }
23178
23179    async fn run(
23180        jobs: Vec<SeatJob>,
23181        sem: Arc<Semaphore>,
23182        cache: Option<&Path>,
23183        st: &mut RunState,
23184    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23185        let prompts = Prompts::default();
23186        let ctx = WaveCtx {
23187            run: "r",
23188            node: "implement",
23189            prompts: &prompts,
23190            cache,
23191            round: None,
23192            carry_seats: false,
23193            pauses: None,
23194        };
23195        wave(jobs, sem, &ctx, st, 0).await
23196    }
23197
23198    #[tokio::test]
23199    async fn verification_cache_wait_preserves_the_command_budget() {
23200        crate::run::pin_test_home();
23201        let tmp = tempfile::tempdir().unwrap();
23202        let cache = tmp.path().join("target");
23203        let guard = hold(&cache);
23204        let release = tokio::spawn(async move {
23205            tokio::time::sleep(Duration::from_millis(100)).await;
23206            guard.release();
23207        });
23208        let mut st = state(10);
23209        let budget = Duration::from_millis(50);
23210        let wait = repository_wait(&st, None);
23211        let (outcomes, _) = with_cache_lease(
23212            &mut st,
23213            Some(&cache),
23214            "gate",
23215            "gate",
23216            tmp.path(),
23217            "abc1234",
23218            budget,
23219            "test gate",
23220            &wait,
23221            |_, available| async move {
23222                assert_eq!(
23223                    available, budget,
23224                    "contention must not shorten verification"
23225                );
23226                (Vec::new(), false, Vec::new())
23227            },
23228        )
23229        .await
23230        .unwrap();
23231        release.await.unwrap();
23232        assert!(outcomes.is_empty(), "{outcomes:?}");
23233        assert!(
23234            st.events
23235                .iter()
23236                .any(|event| event.message.contains("waiting for"))
23237        );
23238        assert!(st.resource_wait.is_none());
23239        // The verification body no longer owns the lease after returning.
23240        hold(&cache).release();
23241    }
23242
23243    #[tokio::test]
23244    async fn verification_cache_wait_expires_without_starting_commands() {
23245        crate::run::pin_test_home();
23246        let tmp = tempfile::tempdir().unwrap();
23247        let cache = tmp.path().join("target");
23248        let _held = hold(&cache);
23249        let mut st = state(0);
23250        let wait = repository_wait(&st, None);
23251        let (outcomes, _) = with_cache_lease(
23252            &mut st,
23253            Some(&cache),
23254            "gate",
23255            "gate",
23256            tmp.path(),
23257            "abc1234",
23258            Duration::from_secs(60),
23259            "test gate",
23260            &wait,
23261            |_, _| async { panic!("commands must not start without the cache lease") },
23262        )
23263        .await
23264        .unwrap();
23265        assert_eq!(outcomes.len(), 1);
23266        assert!(outcomes[0].resource_blocked);
23267        assert!(st.resource_wait.is_none());
23268        assert!(matches!(
23269            st.resource_block.unwrap().cause,
23270            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23271        ));
23272    }
23273
23274    #[tokio::test]
23275    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23276        crate::run::pin_test_home();
23277        let tmp = tempfile::tempdir().unwrap();
23278        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23279        let cache = tmp.path().join("cache");
23280        let mut state = state(21600);
23281        let wait = repository_wait(&state, None);
23282        let (outcomes, _) = with_cache_lease(
23283            &mut state,
23284            Some(&cache),
23285            "gate",
23286            "gate",
23287            tmp.path(),
23288            "head",
23289            Duration::from_secs(60),
23290            "gate",
23291            &wait,
23292            |_, _| async { panic!("freshness failure must not start verification") },
23293        )
23294        .await
23295        .unwrap();
23296        assert_eq!(outcomes.len(), 1);
23297        assert!(outcomes[0].resource_blocked);
23298        let block = state.resource_block.unwrap();
23299        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23300        assert!(!block.description().contains("timed out"));
23301    }
23302
23303    #[tokio::test]
23304    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23305        crate::run::pin_test_home();
23306        let tmp = tempfile::tempdir().unwrap();
23307        let cache = tmp.path().join("cache");
23308        let _held = hold(&cache);
23309        let mut state = state(21600);
23310        let pause = Pause::new();
23311        let interrupt = Pause::new();
23312        let cause = ParkCause::Operator;
23313        pause.park_for(cause.clone());
23314        let prompts = Prompts::default();
23315        let ctx = WaveCtx {
23316            run: "test",
23317            node: "fix",
23318            prompts: &prompts,
23319            cache: Some(&cache),
23320            round: None,
23321            carry_seats: false,
23322            pauses: Some((&pause, &interrupt)),
23323        };
23324        let result = wave(
23325            vec![job(
23326                sh("writer", "exit 99"),
23327                tmp.path(),
23328                Duration::from_secs(1),
23329            )],
23330            Arc::new(Semaphore::new(1)),
23331            &ctx,
23332            &mut state,
23333            0,
23334        )
23335        .await;
23336        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23337        assert_eq!(result[0].1.turns, 0);
23338        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23339        assert_eq!(
23340            state.resource_block.unwrap().cause,
23341            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23342        );
23343        assert!(state.active.is_empty());
23344    }
23345
23346    #[tokio::test]
23347    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23348        crate::run::pin_test_home();
23349        let tmp = tempfile::tempdir().unwrap();
23350        let cache = tmp.path().join("target-a");
23351        let guard = hold(&cache);
23352        // Held for longer than the agent's whole 1s budget.
23353        tokio::spawn(async move {
23354            tokio::time::sleep(Duration::from_millis(1500)).await;
23355            guard.release();
23356        });
23357        let mut st = state(60);
23358        let results = run(
23359            vec![job(
23360                sh("alpha", "echo ok"),
23361                tmp.path(),
23362                Duration::from_secs(1),
23363            )],
23364            Arc::new(Semaphore::new(1)),
23365            Some(&cache),
23366            &mut st,
23367        )
23368        .await;
23369        assert!(
23370            matches!(results[0].2, AgentOutcome::Ok(_)),
23371            "the wait must not be charged to the agent"
23372        );
23373        assert!(st.resource_block.is_none());
23374        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23375        assert!(st.resource_wait.is_none() && st.active.is_empty());
23376    }
23377
23378    #[tokio::test]
23379    #[cfg(unix)]
23380    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23381        use std::os::fd::AsRawFd as _;
23382        crate::run::pin_test_home();
23383        let tmp = tempfile::tempdir().unwrap();
23384        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23385        let common = tmp.path().join(".git");
23386        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23387        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23388        assert_eq!(
23389            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23390            0
23391        );
23392        let mut st = state(1);
23393        st.config.coordination = crate::config::test_coordination();
23394        let mut reader = job(
23395            sh("reader", "echo must-not-run"),
23396            tmp.path(),
23397            Duration::from_secs(30),
23398        );
23399        reader.allow_write = false;
23400        let results = run(
23401            vec![reader.clone()],
23402            Arc::new(Semaphore::new(1)),
23403            None,
23404            &mut st,
23405        )
23406        .await;
23407        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23408        assert_eq!(results[0].1.turns, 0);
23409        assert!(FailClass::of(&results[0].2).is_none());
23410        assert_eq!(
23411            st.resource_block.as_ref().unwrap().kind,
23412            "repository coordination"
23413        );
23414        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23415        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23416        drop(external);
23417        let editor = crate::repo_lock::SeatLease::for_checkout(
23418            tmp.path(),
23419            true,
23420            crate::config::test_coordination().as_ref(),
23421        )
23422        .await
23423        .unwrap();
23424        let pause = Pause::new();
23425        let interrupt = Pause::new();
23426        let prompts = Prompts::default();
23427        let ctx = WaveCtx {
23428            run: "r",
23429            node: "review",
23430            prompts: &prompts,
23431            cache: None,
23432            round: None,
23433            carry_seats: false,
23434            pauses: Some((&pause, &interrupt)),
23435        };
23436        let (results, ()) = tokio::join!(
23437            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23438            async {
23439                tokio::time::sleep(Duration::from_millis(50)).await;
23440                pause.park();
23441            }
23442        );
23443        assert!(
23444            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23445        );
23446        assert_eq!(results[0].1.turns, 0);
23447        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23448        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23449        drop(editor);
23450    }
23451
23452    #[tokio::test]
23453    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23454        crate::run::pin_test_home();
23455        let tmp = tempfile::tempdir().unwrap();
23456        let cache = tmp.path().join("target-b");
23457        let _held = hold(&cache);
23458        let mut st = state(1);
23459        let results = run(
23460            vec![job(
23461                sh("alpha", "echo ok"),
23462                tmp.path(),
23463                Duration::from_secs(1),
23464            )],
23465            Arc::new(Semaphore::new(1)),
23466            Some(&cache),
23467            &mut st,
23468        )
23469        .await;
23470        let (_, seat, out) = &results[0];
23471        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23472        assert!(wave_blocked(&results));
23473        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23474        assert_eq!(seat.turns, 0, "nothing was asked");
23475        assert!(
23476            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23477            "no agent was spawned"
23478        );
23479        assert!(FailClass::of(out).is_none(), "not a failure class");
23480        let block = st.resource_block.as_ref().expect("recorded for the resume");
23481        assert_eq!(block.kind, "shared build cache");
23482        assert!(matches!(
23483            block.cause,
23484            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23485        ));
23486        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23487        assert!(st.active.is_empty());
23488    }
23489
23490    #[tokio::test]
23491    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23492        crate::run::pin_test_home();
23493        let tmp = tempfile::tempdir().unwrap();
23494        let mut st = state(1);
23495        let results = run(
23496            vec![job(
23497                sh("alpha", "echo ok"),
23498                tmp.path(),
23499                Duration::from_secs(1),
23500            )],
23501            Arc::new(Semaphore::new(0)),
23502            None,
23503            &mut st,
23504        )
23505        .await;
23506        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23507        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23508        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23509    }
23510
23511    #[tokio::test]
23512    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23513        crate::run::pin_test_home();
23514        let tmp = tempfile::tempdir().unwrap();
23515        let mut st = state(60);
23516        let results = run(
23517            vec![job(
23518                sh("alpha", "sleep 30"),
23519                tmp.path(),
23520                Duration::from_secs(1),
23521            )],
23522            Arc::new(Semaphore::new(1)),
23523            None,
23524            &mut st,
23525        )
23526        .await;
23527        let out = &results[0].2;
23528        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23529        let class = FailClass::of(out).expect("a failure");
23530        assert_eq!(class, FailClass::Timeout);
23531        assert!(should_hand_over(None, &class), "existing handover policy");
23532        assert!(st.resource_block.is_none());
23533    }
23534}