Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615impl Runner {
616    /// Start a fresh run against `repo`.
617    pub async fn start(
618        repo: &Path,
619        instruction: String,
620        config: Config,
621        origin: Origin,
622    ) -> Result<Self> {
623        Self::start_naming(repo, instruction, "", config, origin).await
624    }
625
626    /// [`Runner::start`] for a queued task. Titles and instructions never
627    /// select implementation sources. The title argument is retained for callers.
628    pub async fn start_naming(
629        repo: &Path,
630        instruction: String,
631        _also_scan: &str,
632        config: Config,
633        origin: Origin,
634    ) -> Result<Self> {
635        let repo = git::toplevel(repo).await?;
636        let missing = agent::missing_programs(&config.agents);
637        if !missing.is_empty() {
638            bail!(
639                "these agent programs are not on PATH: {}. Fix the roster in \
640                 magi.toml or install them.",
641                missing.join(", ")
642            );
643        }
644        let base_branch =
645            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646                .await?;
647        let fetched_base = resolve_base(
648            &repo,
649            &base_branch,
650            &config.merge.remote,
651            config.coordination.as_ref(),
652        )
653        .await?;
654        let base_commit = fetched_base.commit.clone();
655        let roles = config.resolve_roles()?;
656        let max_parallel = config.graph.max_parallel.max(1);
657        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658        // Recorded before the first save, so a crash right after minting
659        // cannot leave a run with no origin. Legibility only: nothing reads it
660        // to decide anything.
661        state.fetched_base = Some(fetched_base);
662        state.origin = Some(origin);
663        state.reserve()?;
664        state.event("start", format!("run {} created", state.id));
665        state.save()?;
666        Ok(Self {
667            state,
668            roles,
669            sem: Arc::new(Semaphore::new(max_parallel)),
670            pause: Pause::new(),
671            interrupt: Pause::new(),
672        })
673    }
674
675    /// Open a review-only run against work that already exists on `branch`.
676    ///
677    /// The expensive half of the graph is the implement wave — measured at
678    /// 111 and 134 internal tool-loop turns on this repository, against a
679    /// handful for a judge or a reviewer. The cheap half is worth running on
680    /// hand-written work too, and there was no way to reach it.
681    ///
682    /// No new state and no schema change are needed: a run with **one** viable
683    /// candidate and a tally already decided degrades `execute` to exactly
684    /// review → gate → merge, because `judge` skips a single-candidate field,
685    /// `deliberate` has fewer than two first choices to reconcile, `vote`
686    /// returns early, `tally` is already present and `fold_losers` has no
687    /// losers. Resuming such a run therefore does the right thing as well.
688    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689        Self::review_taking_over(repo, branch, config, None, origin).await
690    }
691
692    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
693    /// at the same task still has `branch` checked out, its worktree is
694    /// released first if that is safe (see [`crate::handover`]), and the
695    /// review refuses with the reason if it is not. `None` is a hand-run
696    /// review: it has no earlier attempts, so only a worktree of a dead run
697    /// magi recorded itself can be released.
698    pub async fn review_taking_over(
699        repo: &Path,
700        branch: &str,
701        config: Config,
702        takeover: Option<crate::handover::Takeover>,
703        origin: Origin,
704    ) -> Result<Self> {
705        let repo = git::toplevel(repo).await?;
706        let missing = agent::missing_programs(&config.agents);
707        if !missing.is_empty() {
708            bail!(
709                "these agent programs are not on PATH: {}. Fix the roster in \
710                 magi.toml or install them.",
711                missing.join(", ")
712            );
713        }
714        let base_branch =
715            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716                .await?;
717        if base_branch == branch {
718            bail!("`{branch}` is the base branch; there is nothing to review against");
719        }
720        let fetched_base = resolve_base(
721            &repo,
722            &base_branch,
723            &config.merge.remote,
724            config.coordination.as_ref(),
725        )
726        .await?;
727        let base_commit = fetched_base.commit.clone();
728
729        let roles = config.resolve_roles()?;
730        let max_parallel = config.graph.max_parallel.max(1);
731        let mut state = RunState::new(
732            repo.clone(),
733            base_branch,
734            base_commit.clone(),
735            String::new(),
736            config,
737        );
738        state.fetched_base = Some(fetched_base);
739        state.origin = Some(origin);
740        state.reserve()?;
741
742        // Released before anything else touches the branch: a stale local
743        // branch is moved with `git branch -f`, which git refuses while an
744        // earlier attempt's worktree still has it checked out. Everything
745        // after this point that can fail puts the old run back.
746        // A hand-run review has no task, hence no earlier attempts, but a
747        // worktree of a dead run magi made may still be released.
748        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749            earlier: Vec::new(),
750            home: crate::run::home(),
751            choice: None,
752            task_instruction: None,
753        });
754        let wait = repository_wait(&state, None);
755        let coordination_config = state.config.coordination.clone();
756        let coordination =
757            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758                .await?;
759        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760        if let Some(released) = &released {
761            state.event(
762                "release",
763                format!(
764                    "took `{branch}` over from run {}: its worktree was released: {}",
765                    crate::run::short_of(&released.old_id),
766                    released.audit
767                ),
768            );
769        }
770        // The owner's answer to an earlier divergence question is applied
771        // here: after the release (git will not move a checked-out branch)
772        // and before the sync that would otherwise ask again.
773        if let Some(choice) = takeover.choice.as_ref()
774            && let Err(e) =
775                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
776                    .await
777        {
778            if let Some(released) = &released {
779                released.restore(&repo, branch).await;
780            }
781            return Err(e.context("applying the owner's answer about the diverged branch"));
782        }
783        // An earlier attempt of this task that published the task head, and
784        // whose work is the branch under review, hands that head over: the
785        // review run updates the same pull request. Without one (a legacy
786        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
787        // own head and its own open pull request is adopted as before.
788        for id in takeover.earlier.iter().rev() {
789            if let Ok(old) = RunState::load_under(id, &takeover.home)
790                && let Some(head) = old.pr_head.clone()
791                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
792            {
793                state.pr_head = Some(head);
794                break;
795            }
796        }
797        drop(coordination);
798        let opened = Self::open_review(
799            &repo,
800            branch,
801            state,
802            roles,
803            max_parallel,
804            base_commit,
805            &takeover,
806        )
807        .await;
808        if opened.is_err()
809            && let Some(released) = &released
810        {
811            let _coordination = crate::repo_lock::SeatLease::operation_with(
812                &repo,
813                coordination_config.as_ref(),
814                &wait,
815            )
816            .await?;
817            released.restore(&repo, branch).await;
818        }
819        opened
820    }
821
822    /// Recover exactly the operator-selected public implementation, without touching old sources.
823    pub async fn recover_pr(
824        repo: &Path,
825        task: &str,
826        selection: crate::pr_recovery::Selection,
827        mut config: Config,
828    ) -> Result<Self> {
829        selection.configure(&mut config)?;
830        selection.check_task(task, &selection.instruction)?;
831        let base_branch =
832            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
833                .await?;
834        let fetched_base = resolve_base(
835            repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842        selection
843            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
844            .await?;
845        Box::pin(selection.verify_source(repo)).await?;
846        if let Some(proof) = &selection.source_proof {
847            anyhow::ensure!(
848                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
849                "source rebase base is outside the fetched landing base"
850            );
851        }
852        let roles = config.resolve_roles()?;
853        let max_parallel = config.graph.max_parallel.max(1);
854        let mut state = RunState::new(
855            repo.to_path_buf(),
856            base_branch,
857            base_commit.clone(),
858            selection.instruction.clone(),
859            config,
860        );
861        state.fetched_base = Some(fetched_base);
862        state.origin = Some(crate::run::Origin::queue(task));
863        state.pr_head = Some(selection.branch.clone());
864        state.pr_recovery = Some(selection.clone());
865        state.reserve()?;
866        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
867            // Never leave a reserved record behind as a phantom owner of the head.
868            state.discard_reserved();
869            return Err(e);
870        }
871        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
872    }
873
874    async fn open_selected(
875        repo: &Path,
876        state: RunState,
877        roles: ResolvedRoles,
878        max_parallel: usize,
879        base_commit: String,
880        selection: crate::pr_recovery::Selection,
881    ) -> Result<Self> {
882        let mut state = state;
883        let branch = state.branch_for('A');
884        let wait = repository_wait(&state, None);
885        let prepared = async {
886            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
887            git::git(repo, &["branch", &branch, selection.source()]).await?;
888            drop(coordination);
889            anyhow::Ok(())
890        }
891        .await;
892        if let Err(e) = prepared {
893            // Nothing was exposed yet: do not leave a reserved record as a
894            // phantom owner of the public head.
895            state.discard_reserved();
896            return Err(e);
897        }
898        state.event(
899            "recovery",
900            format!(
901                "selected implementation {} for {} (public lease {})",
902                selection.source(),
903                selection.url,
904                selection.head
905            ),
906        );
907        let takeover = crate::handover::Takeover {
908            earlier: Vec::new(),
909            home: crate::run::home(),
910            choice: None,
911            task_instruction: Some(selection.instruction.clone()),
912        };
913        if let Err(e) = state.save() {
914            state.discard_reserved();
915            return Err(e);
916        }
917        let id = state.id.clone();
918        let opened = Self::open_review(
919            repo,
920            &branch,
921            state,
922            roles,
923            max_parallel,
924            base_commit,
925            &takeover,
926        )
927        .await;
928        match opened {
929            Ok(runner) => Ok(runner),
930            Err(e) => {
931                if let Ok(mut partial) = RunState::load(&id) {
932                    partial.status = RunStatus::Blocked;
933                    partial.driver_exited = true;
934                    partial.event(
935                        "recovery",
936                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
937                    );
938                    partial.save()?;
939                }
940                Err(crate::pr_recovery::StartupFailed {
941                    run: id,
942                    cause: format!("{e:#}"),
943                }
944                .into())
945            }
946        }
947    }
948
949    /// What the newest earlier attempt that reviewed `branch` leaves for this
950    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
951    /// no earlier attempts. Reads the earlier record only: ownership of its
952    /// worktree and branch is [`crate::handover`]'s business.
953    async fn inherit(
954        repo: &Path,
955        branch: &str,
956        base_commit: &str,
957        takeover: &crate::handover::Takeover,
958    ) -> Option<crate::recovery::RecoveryRecord> {
959        use crate::recovery::{Relation, assess};
960        let newest = takeover.earlier.last()?;
961        let mut chosen = None;
962        for id in takeover.earlier.iter().rev() {
963            if let Ok(old) = RunState::load_under(id, &takeover.home)
964                && !old.reviews.is_empty()
965                && (old.winner().is_some_and(|w| w.branch == branch)
966                    || old.pr_head.as_deref() == Some(branch))
967            {
968                chosen = Some(old);
969                break;
970            }
971        }
972        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
973            .await
974            .unwrap_or_default();
975        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
976            Some(last) if last.head == tip => Relation::Same,
977            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
978            _ => Relation::Diverged,
979        };
980        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
981        Some(assess(from, chosen.as_ref(), base_commit, relation))
982    }
983
984    /// The half of [`Runner::review_taking_over`] that can fail after an
985    /// earlier attempt's worktree was released.
986    async fn open_review(
987        repo: &Path,
988        branch: &str,
989        mut state: RunState,
990        roles: ResolvedRoles,
991        max_parallel: usize,
992        base_commit: String,
993        takeover: &crate::handover::Takeover,
994    ) -> Result<Self> {
995        let wait = repository_wait(&state, None);
996        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
997        if state.pr_recovery.is_none() {
998            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
999        }
1000        // The commit subjects are the closest thing to a task statement that
1001        // existing work carries, and the reviewers are told as much.
1002        let start = review_base(
1003            repo,
1004            &state.config.merge.remote,
1005            &state.base_branch,
1006            &base_commit,
1007            branch,
1008        )
1009        .await;
1010        let log = git::log_oneline(repo, &start, branch)
1011            .await
1012            .unwrap_or_default();
1013        let log_text = if log.trim().is_empty() {
1014            "(no commit messages)"
1015        } else {
1016            log.trim()
1017        };
1018        let mut instruction = format!(
1019            "Review the work already on branch `{branch}`. There is no task \
1020             statement: what the change claims to do is whatever its commits \
1021             say.\n\n{log_text}"
1022        );
1023        // A retry of a task whose earlier attempt left reviews behind carries
1024        // that attempt's evidence, and the task's own statement with it: the
1025        // completion criteria are the task's, not whatever the commits claim.
1026        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1027            if let Some(task) = takeover
1028                .task_instruction
1029                .as_deref()
1030                .filter(|t| !t.trim().is_empty())
1031                && rec.reviewed_head.is_some()
1032            {
1033                instruction = format!(
1034                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1035                    task.trim()
1036                );
1037                rec.criteria_inherited = true;
1038            }
1039            state.event("recovery", rec.summary());
1040            state.recovery = Some(rec);
1041        }
1042        if let Some(selected) = &state.pr_recovery {
1043            instruction = selected.instruction.clone();
1044        }
1045        state.task_instruction = takeover.task_instruction.clone();
1046        state.instruction = instruction;
1047        state.reviewed_commits = Some(
1048            git::subjects(repo, &start, branch)
1049                .await
1050                .unwrap_or_default(),
1051        );
1052
1053        // A takeover must not forget a hold that a previous review observed,
1054        // even if repairs have since removed it from the branch.
1055        for id in &takeover.earlier {
1056            if let Ok(old) = RunState::load_under(id, &takeover.home)
1057                && old.repo == state.repo
1058                && (old.pr_head.as_deref() == Some(branch)
1059                    || old.winner().is_some_and(|w| w.branch == branch))
1060            {
1061                for hold in old.branch_owner_holds {
1062                    if !state.branch_owner_holds.contains(&hold) {
1063                        state.event(
1064                            "approval-policy",
1065                            format!(
1066                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1067                                hold.control.key(),
1068                                hold.file,
1069                                hold.head
1070                            ),
1071                        );
1072                        state.branch_owner_holds.push(hold);
1073                    }
1074                }
1075            }
1076        }
1077        let reviewed_head = git::rev_parse(repo, branch).await?;
1078        state.observe_branch_owner_holds(&reviewed_head).await;
1079
1080        // An attached worktree, so the fixer's commits land on the branch under
1081        // review rather than on a detached head nobody will look at again.
1082        let worktree = state.worktree_root().join("under-review");
1083        if let Some(parent) = worktree.parent() {
1084            tokio::fs::create_dir_all(parent).await.ok();
1085        }
1086        let path = worktree.to_string_lossy().to_string();
1087        git::git(repo, &["worktree", "add", &path, branch])
1088            .await
1089            .with_context(|| {
1090                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1091            })?;
1092
1093        let setup = match ensure_setup_config(&mut state, repo).await {
1094            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1095            Err(e) => Err(e),
1096        };
1097        if let Err(e) = setup {
1098            git::worktree_remove_clean(repo, &worktree).await.ok();
1099            return Err(e);
1100        }
1101        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1102            .await
1103            .unwrap_or(0);
1104        if commits == 0 {
1105            git::worktree_remove_clean(repo, &worktree).await.ok();
1106            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1107        }
1108        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1109            .await
1110            .map(|f| f.len())
1111            .unwrap_or(0);
1112        if files == 0
1113            && let (Ok(head_tree), Ok(base_tree)) = (
1114                git::tree_of(&worktree, "HEAD").await,
1115                git::tree_of(&worktree, &base_commit).await,
1116            )
1117            && head_tree == base_tree
1118        {
1119            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1120            git::worktree_remove_clean(repo, &worktree).await.ok();
1121            bail!(
1122                "`{branch}` at {} has a tree identical to base {}; this usually means \
1123                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1124                 against `{}/{branch}`) rather than an empty change",
1125                short(&head),
1126                short(&base_commit),
1127                state.config.merge.remote
1128            );
1129        }
1130        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1131            .await
1132            .unwrap_or_default();
1133
1134        state.candidates.push(Candidate {
1135            index: 0,
1136            label: 'A',
1137            // Not an agent id on purpose: nothing in the roster wrote this, and
1138            // the stats tables must not credit anyone with a win for it.
1139            agent: EXISTING_BRANCH.to_owned(),
1140            branch: branch.to_owned(),
1141            worktree,
1142            summary: String::new(),
1143            stat,
1144            files,
1145            commits,
1146            empty: false,
1147            failed: None,
1148            verified_noop: None,
1149            duration_ms: 0,
1150            folded: false,
1151        });
1152        state.tally = Some(Tally {
1153            first_choice: BTreeMap::from([('A', 0)]),
1154            borda: BTreeMap::new(),
1155            winner: 'A',
1156            rankings: 0,
1157            unanimous_initial: false,
1158            deliberated: false,
1159            changed_votes: 0,
1160            unanimous_final: false,
1161            tie_break: None,
1162            // No panel sat, so no quorum applies. Zero judges is the correct
1163            // number for work that never competed, and must not be reported as
1164            // a collapsed panel.
1165            judges: 0,
1166            present: 0,
1167            quorum: 0,
1168            met_quorum: true,
1169            uncontested: Some("review-only run: nothing competed".to_owned()),
1170        });
1171        state.status = RunStatus::Reviewing;
1172        state.prep_complete = Some(true);
1173        state.event(
1174            "start",
1175            format!(
1176                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1177                state.id
1178            ),
1179        );
1180        state.save()?;
1181        Ok(Self {
1182            state,
1183            roles,
1184            sem: Arc::new(Semaphore::new(max_parallel)),
1185            pause: Pause::new(),
1186            interrupt: Pause::new(),
1187        })
1188    }
1189
1190    /// Authorize exactly one additional rebase, under the caller's queue task
1191    /// claim and the same owner/execution locks used by run maintenance.
1192    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1193        Self::authorize_base_recovery_request(home, id, task, false).await
1194    }
1195
1196    /// Queue release uses its durable approval generation exactly once.
1197    pub(crate) async fn authorize_base_recovery_from_resumption(
1198        home: &Path,
1199        id: &str,
1200        task: &str,
1201    ) -> Result<()> {
1202        Self::authorize_base_recovery_request(home, id, task, true).await
1203    }
1204
1205    async fn authorize_base_recovery_request(
1206        home: &Path,
1207        id: &str,
1208        task: &str,
1209        resumption: bool,
1210    ) -> Result<()> {
1211        let _owner = crate::pr_head::owner_lock(home, id)?;
1212        let mut state = RunState::load_under(id, home)?;
1213        crate::pr_head::ensure_owner(&state)?;
1214        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1215        anyhow::ensure!(
1216            queued.runs.last().is_some_and(|run| run == id)
1217                && queued.status != crate::queue::TaskStatus::Done
1218                && !queued.fresh_start
1219                && queued.review_branch.is_none(),
1220            "run task ownership changed"
1221        );
1222        anyhow::ensure!(
1223            state
1224                .origin
1225                .as_ref()
1226                .and_then(|o| o.task.as_deref())
1227                .is_none_or(|owner| owner == task),
1228            "run belongs to another task"
1229        );
1230        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1231        anyhow::ensure!(
1232            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1233            "run is live or its ownership is unknown"
1234        );
1235        // Judge the previous driver's lease before acquiring our maintenance
1236        // lease. The owner lock excludes new driver claims across both steps.
1237        let _execution = crate::driver::maintenance(home, id)?;
1238        anyhow::ensure!(
1239            state.status.resumable()
1240                && state.released_to.is_none()
1241                && state.retired_to.is_none()
1242                && state.pr_taken_over_by.is_none(),
1243            "run ownership is final"
1244        );
1245        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1246        anyhow::ensure!(
1247            !state.pr_publication_pending
1248                && !crate::handover::landing_checkpoint(&state)
1249                && state.merge_decision.is_none(),
1250            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1251        );
1252        let winner = state.winner().context("run has no retained winner")?;
1253        anyhow::ensure!(
1254            git::is_clean(&winner.worktree).await?,
1255            "retained worktree has uncommitted changes"
1256        );
1257        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1258        anyhow::ensure!(
1259            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1260            "retained worktree does not match the winner branch"
1261        );
1262        state.record_base_sync_stop(&head);
1263        state.save()?;
1264        if resumption {
1265            anyhow::ensure!(
1266                crate::daemon::compatible_run_instruction(&state, &queued),
1267                "resumption instruction does not match the retained run"
1268            );
1269            let approval = queued
1270                .resumption
1271                .as_ref()
1272                .context("no operator resumption")?;
1273            anyhow::ensure!(
1274                approval.instruction == queued.instruction,
1275                "resumption approval does not match the current task instruction"
1276            );
1277            state.authorize_base_sync_resumption(task, approval)?;
1278        } else {
1279            state.authorize_base_sync(task)?;
1280        }
1281        state.event(
1282            "base-recovery",
1283            "owner authorized one additional base rebase; fresh verification required",
1284        );
1285        state.save()?;
1286        Ok(())
1287    }
1288
1289    /// Reopen an existing run.
1290    pub fn resume(id: &str) -> Result<Self> {
1291        let state = RunState::load(id)?;
1292        crate::pr_head::ensure_owner(&state)?;
1293        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1294            bail!(
1295                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1296            );
1297        }
1298        if let Some(to) = &state.released_to {
1299            bail!(
1300                "run {} cannot be resumed: its worktree was released to run {}",
1301                state.short(),
1302                crate::run::short_of(to)
1303            );
1304        }
1305        let roles = state.config.resolve_roles()?;
1306        if let Some(id) = &state.config.graph.selected_implementer
1307            && state.candidates.iter().any(|c| c.agent != *id)
1308        {
1309            bail!(
1310                "replacement implementer pin does not match the saved candidates; task must stay held"
1311            );
1312        }
1313        let max_parallel = state.config.graph.max_parallel.max(1);
1314        Ok(Self {
1315            state,
1316            roles,
1317            sem: Arc::new(Semaphore::new(max_parallel)),
1318            pause: Pause::new(),
1319            interrupt: Pause::new(),
1320        })
1321    }
1322
1323    /// Walk the graph to a terminal state, skipping nodes already recorded.
1324    ///
1325    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1326    /// the phone - ends here, so this is the one place a run that ended
1327    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1328    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1329    pub async fn execute(&mut self) -> Result<()> {
1330        if let Some(selection) = &self.state.pr_recovery
1331            && (selection.source_proof.is_some() || selection.source_run.is_some())
1332        {
1333            Box::pin(selection.verify_source(&self.state.repo)).await?;
1334        }
1335        // Queue claims cover daemon work; this also serializes manual drivers
1336        // consuming a repair answer, including the new review and decision.
1337        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1338            || self.state.land_approval.is_some()
1339            || self.state.merge_decision.is_some()
1340            || !self.state.merge_repairs.is_empty()
1341            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1342        {
1343            let claim = FixClaim::acquire(&self.state.dir())?;
1344            let disk = RunState::load(&self.state.id)?;
1345            let checkpoint = |s: &RunState| {
1346                (
1347                    s.merge_repairs.len(),
1348                    s.merge_repairs
1349                        .last()
1350                        .map(|r| (r.question.clone(), r.phase)),
1351                    s.land_approval
1352                        .as_ref()
1353                        .map(|a| (a.question.clone(), a.head.clone())),
1354                )
1355            };
1356            if checkpoint(&disk) != checkpoint(&self.state) {
1357                bail!(
1358                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1359                );
1360            }
1361            Some(claim)
1362        } else {
1363            None
1364        };
1365        let mut execution = self.claim_driver().await?;
1366        let runner = &mut *execution.runner;
1367        let wait = runner.repository_wait();
1368        // Keep the graph's large future off the callers' stack, including
1369        // standalone task attempts running on a default-sized test thread.
1370        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1371        let result = match result {
1372            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1373                record_repository_block(&mut runner.state, &e);
1374                runner.park_here()?;
1375                Ok(())
1376            }
1377            other => other,
1378        };
1379        execution.driver.finish(&mut runner.state).await?;
1380        let ended = if result.is_err() {
1381            Some({
1382                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1383                if let Err(error) = &result {
1384                    notice.message = format!(
1385                        "Run {} stopped with an error: {}",
1386                        runner.state.id,
1387                        error.to_string().lines().next().unwrap_or("unknown error")
1388                    );
1389                }
1390                notice
1391            })
1392        } else {
1393            crate::notices::run_ended(&runner.state)
1394        };
1395        if let Some(notice) = ended {
1396            crate::notices::raise(notice);
1397        }
1398        result
1399    }
1400
1401    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1402        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1403    }
1404
1405    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1406        let wait = self.repository_wait();
1407        coordinate_operation(&mut self.state, cwd, &wait).await
1408    }
1409
1410    /// Validate ownership and persist the driver claim in one critical section.
1411    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1412        let owner = match crate::pr_head::owner_lock_wait(
1413            &crate::run::home(),
1414            &self.state.id,
1415            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1416        )
1417        .await
1418        {
1419            Ok(owner) => owner,
1420            Err(error) => {
1421                let detail = format!("worktree maintenance or ownership update: {error:#}");
1422                // Admission did not own run.json: retain this resource outcome
1423                // in memory for queue settlement without overwriting a concurrent
1424                // retirement or publication transaction.
1425                self.state.resource_block = Some(crate::run::ResourceBlock {
1426                    cause: crate::run::ResourceBlockCause::Other,
1427                    node: "start".into(),
1428                    kind: "worktree maintenance".into(),
1429                    detail: detail.clone(),
1430                    at: Timestamp::now(),
1431                });
1432                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1433            }
1434        };
1435        let disk = RunState::load(&self.state.id)?;
1436        crate::pr_head::ensure_owner(&disk)?;
1437        let claimed =
1438            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1439        // Only proven-live ownership refuses here. Unknown records (legacy runs
1440        // with no claim marker or process identity) fall through to the
1441        // exclusive execution lease taken just below, which is what actually
1442        // keeps a second driver out.
1443        anyhow::ensure!(
1444            disk.liveness(claimed) != crate::run::Liveness::Live,
1445            "run already has a live driver; wait for its execution to end"
1446        );
1447        if let Some(to) = &disk.released_to {
1448            bail!(
1449                "run {} cannot continue: its worktree was released to run {}",
1450                self.state.short(),
1451                crate::run::short_of(to)
1452            );
1453        }
1454        anyhow::ensure!(
1455            serde_json::to_value(&disk.base_sync_recovery)?
1456                == serde_json::to_value(&self.state.base_sync_recovery)?,
1457            "base recovery changed since this run was loaded; reload the saved run"
1458        );
1459        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1460        let mut execution = DriverExecution {
1461            runner: self,
1462            driver,
1463        };
1464        // Declare after the execution guard: on any claim error this short
1465        // lock must drop before that guard reloads and releases ownership.
1466        let _owner = owner;
1467        let state = &mut execution.runner.state;
1468        // Moving again, so it is no longer parked. Set before the walk rather
1469        // than in `resume`, so every way of re-entering the graph clears it
1470        // and a card cannot claim a run is waiting to be resumed while the
1471        // agents are already working.
1472        state.parked = false;
1473        state.resource_wait = None;
1474        state.resource_block = None;
1475        // Any seat this state still lists as answering belongs to whatever
1476        // process last drove this run — this one included, if it crashed
1477        // mid-wave. Cleared and flushed immediately, before anything else
1478        // runs, so a resume can never show a seat as live when nothing is
1479        // asking it anything yet; the node that actually dispatches the next
1480        // wave repopulates it.
1481        state.clear_active();
1482        // Recorded in the same spot, and flushed together with the clear
1483        // above: this is the pid a reader checks (`RunState::liveness`) when
1484        // no daemon claim exists to answer "is a process still driving this
1485        // run" — a plain `magi run` / `magi review` typed into a terminal
1486        // claims nothing there. Always overwritten, never only-if-absent, so
1487        // a resumed run's stale pid from a previous, possibly-dead process
1488        // can never survive into this one's own report. Unlike
1489        // `clear_active`, this changes on every single `execute()` call, so
1490        // the save below is now unconditional rather than only-if-cleared.
1491        //
1492        // `driver_started_at` is recorded in the same breath, from this same
1493        // pid, so `liveness` can tell a live pid that is genuinely still us
1494        // apart from one the OS has since handed to an unrelated process —
1495        // see that field's own doc for why the pid alone is not enough.
1496        let pid = std::process::id();
1497        state.driver_pid = Some(pid);
1498        state.driver_started_at = crate::proc::process_started_at(pid);
1499        state.driver_exited = false;
1500        state.driver_claimed = Some(true);
1501        state.begin_pass();
1502        execution.driver.bind(state);
1503        state.save()?;
1504        execution.driver.record(state)?;
1505        Ok(execution)
1506    }
1507
1508    async fn execute_graph(&mut self) -> Result<()> {
1509        self.state.configure_recovery_resume()?;
1510        // Preservation may invalidate an old empty classification. Resolve it
1511        // before any resume shortcut can reuse a verdict or publish a branch.
1512        self.persist_candidates().await?;
1513        // A saved base stop outranks stage markers, including legacy Voting.
1514        // Recovery never reenters implementation, deliberation or voting.
1515        if self.state.base_sync_blocked() {
1516            self.sync_to_base().await?;
1517            if !self.state.base_sync_blocked()
1518                && self.state.status != RunStatus::AlreadyInBase
1519                && !self.resource_halt()?
1520            {
1521                self.finish_after_tally().await?;
1522            }
1523            return Ok(());
1524        }
1525        if self
1526            .state
1527            .merge_decision
1528            .as_ref()
1529            .is_some_and(|decision| !decision.closed)
1530            && self.state.status != RunStatus::Landing
1531        {
1532            if crate::merge_decision::resumable_hold(&self.state) {
1533                self.state.merge = Some(MergeOutcome {
1534                    mode: MergeMode::Pr,
1535                    ok: false,
1536                    detail: self
1537                        .state
1538                        .merge_decision
1539                        .as_ref()
1540                        .unwrap()
1541                        .target
1542                        .pr
1543                        .clone(),
1544                    empty: false,
1545                });
1546                self.run_land().await?;
1547                self.settle_questions();
1548                return Ok(());
1549            }
1550            bail!(
1551                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1552            );
1553        }
1554        if self.state.config.graph.selected_implementer.is_some()
1555            && self.state.status == RunStatus::Blocked
1556            && self.state.tally.is_none()
1557            && self.state.candidate_preservation.is_empty()
1558            && self.state.interrupted_implementations.is_empty()
1559        {
1560            bail!(
1561                "selected implementation is held; explicitly release the owning task to retry it"
1562            );
1563        }
1564
1565        if self.state.merge_repairs.last().is_some_and(|r| {
1566            matches!(
1567                r.phase,
1568                crate::run::MergeRepairPhase::Requested
1569                    | crate::run::MergeRepairPhase::Fixing
1570                    | crate::run::MergeRepairPhase::Reviewing
1571            )
1572        }) {
1573            if self.repair_merge_review().await? {
1574                self.run_land().await?;
1575            }
1576            self.settle_questions();
1577            return Ok(());
1578        }
1579        // Publication already passed review and gate. Resume only that node:
1580        // syncing to a newer base here could change the verified commit.
1581        if self.state.pr_publication_pending {
1582            self.merge().await?;
1583            if self.resource_halt()? {
1584                return Ok(());
1585            }
1586            self.state.save()?;
1587            self.settle_questions();
1588            return Ok(());
1589        }
1590        // A run that already lost its quorum never resumes into the verdict
1591        // machinery: `deliberate` and `vote` would otherwise clobber the
1592        // stalled marker back to Voting and the run would keep going past a
1593        // verdict that is no longer trustworthy. Everything already recorded is
1594        // kept, so the run stays resumable (or foldable) for a human to pick up.
1595        //
1596        // On --resume the run gets one chance to repair itself: the seats a
1597        // rate limit took out are re-asked. If their quota has since reset and
1598        // the quorum is restored, the run picks up and finishes; otherwise it
1599        // stays stale and still-resumable for a later retry. If it does not
1600        // recover, the returned status stays `Stalled` and nothing was
1601        // clobbered (the recovery only mutates entries for the lost seats).
1602        if self.state.status == RunStatus::Stalled {
1603            if self.recover_stall().await? {
1604                self.finish_after_tally().await?;
1605            } else {
1606                // Still below quorum: persist the marker and stay resumable.
1607                self.state.save()?;
1608            }
1609            return Ok(());
1610        }
1611        // A run parked inside `land` - watching CI, mid fix-round, or
1612        // waiting on the owner's merge approval - resumes directly into it,
1613        // never back through `prep`. Everything before `merge` already
1614        // concluded; that is the only way `status` reaches `Landing` in the
1615        // first place. Re-walking `review_loop` first would also be actively
1616        // wrong: its own status recomputation (see its doc) treats any
1617        // clean round as reason to set `status` to `Gating`, which would
1618        // clobber this marker before `merge` ever ran, and this run would
1619        // never find its way back into `land` at all.
1620        if self.state.status == RunStatus::Landing
1621            || crate::handover::landing_checkpoint(&self.state)
1622        {
1623            self.state.land_sync_pending = true;
1624            self.state.save()?;
1625            self.run_land().await?;
1626            // `run_land` may have settled the run right here - CI came back
1627            // green and the PR merged, say - without ever passing back
1628            // through `merge`'s own trailing call. Whatever it left `status`
1629            // as is what this has to read.
1630            self.settle_questions();
1631            return Ok(());
1632        }
1633        // A run parked at `merge` on the owner's word about a withheld PR text
1634        // resumes straight into `merge`. Re-walking the tail would sync to the
1635        // base again (possibly rebasing, possibly spending the fixer) after the
1636        // gate had passed, and then push a tree nobody reviewed or gated.
1637        if self.state.github_text.is_some()
1638            && self.state.merge.is_none()
1639            && self.state.status == RunStatus::Gating
1640        {
1641            self.merge().await?;
1642            if self.resource_halt()? {
1643                return Ok(());
1644            }
1645            self.state.save()?;
1646            return Ok(());
1647        }
1648        if let Err(e) = self.prep().await {
1649            if e.is::<crate::repo_lock::ResourceBlocked>() {
1650                return Err(e);
1651            }
1652            self.state.status = RunStatus::Blocked;
1653            self.state
1654                .prep_failure
1655                .get_or_insert_with(|| format!("{e:#}"));
1656            self.state
1657                .event("prep", format!("preparation stopped: {e:#}"));
1658            self.state.save()?;
1659            return Err(e);
1660        }
1661        if self.park_here()? {
1662            return Ok(());
1663        }
1664        self.advise().await?;
1665        if self.park_here()? {
1666            return Ok(());
1667        }
1668        self.implement().await?;
1669        if self.park_here()? {
1670            return Ok(());
1671        }
1672        // `after_implement` already saved the state and settled any open
1673        // questions when it set this; nothing later in the graph has
1674        // anything to judge.
1675        if self.state.status == RunStatus::VerifiedNoop {
1676            return Ok(());
1677        }
1678        self.judge().await?;
1679        if self.park_here()? {
1680            return Ok(());
1681        }
1682        self.deliberate().await?;
1683        if self.park_here()? {
1684            return Ok(());
1685        }
1686        self.vote().await?;
1687        if self.park_here()? {
1688            return Ok(());
1689        }
1690        self.tally()?;
1691        // A verdict that lost its quorum is not trustworthy: do not review,
1692        // gate, or merge on it. Everything already done is kept, so the run
1693        // stays resumable (or foldable); the human can replace the agent that
1694        // ran out of quota and pick it up.
1695        if self.state.status == RunStatus::Stalled {
1696            // Persist the stalled marker now — the normal end-of-execute save
1697            // below is below this early return, and without it a resumed run
1698            // would reload a pre-tally status and keep going.
1699            self.state.save()?;
1700            return Ok(());
1701        }
1702        self.finish_after_tally().await?;
1703        Ok(())
1704    }
1705
1706    /// A wave could not start its agents because a shared resource stayed busy
1707    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1708    /// resumable. Unstarted seats have no result; completed peers retain
1709    /// their saved candidate state.
1710    fn stop_for_resource(&mut self) -> Result<()> {
1711        // Resource acquisition observes cancellation inside a node. Complete
1712        // the same durable park as every other node boundary before returning.
1713        if self.pause.parked() || self.interrupt.parked() {
1714            self.park_here()?;
1715            return Ok(());
1716        }
1717        if self.state.resource_block.is_some() {
1718            self.state.status = RunStatus::Blocked;
1719        }
1720        self.state.save()
1721    }
1722
1723    /// Stop walking the graph if a node ended on a resource block.
1724    fn resource_halt(&mut self) -> Result<bool> {
1725        if self.state.resource_block.is_none() {
1726            return Ok(false);
1727        }
1728        self.stop_for_resource()?;
1729        Ok(true)
1730    }
1731
1732    /// Park here if asked to, recording it in the run's own timeline.
1733    ///
1734    /// Returns whether the caller should stop walking the graph. The state is
1735    /// saved either way by the node that just finished; this adds the event so
1736    /// the operator's card says why a run that is neither finished nor moving
1737    /// is sitting where it is.
1738    fn park_here(&mut self) -> Result<bool> {
1739        let parking = self.pause.parked() || self.interrupt.parked();
1740        if !parking && self.state.resource_block.is_some() {
1741            self.stop_for_resource()?;
1742            return Ok(true);
1743        }
1744        // Either handle asking is enough - see `Pause`'s own doc for why
1745        // they are never the same one. `interrupt` is checked second so a
1746        // reason it carries is preferred in the message below over a plain
1747        // shutdown park racing it at the same boundary.
1748        if !self.pause.parked() && !self.interrupt.parked() {
1749            return Ok(false);
1750        }
1751        if self.state.parked {
1752            return Ok(true);
1753        }
1754        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1755        if let Some(block) = &mut self.state.resource_block
1756            && matches!(
1757                block.cause,
1758                crate::run::ResourceBlockCause::Cancelled { .. }
1759            )
1760        {
1761            block.cause = crate::run::ResourceBlockCause::Cancelled {
1762                park: cause.clone(),
1763            };
1764        }
1765        let why = match cause.as_ref().map(ParkCause::label) {
1766            Some(reason) => format!(
1767                "parked after `{}` ({reason}) — resume to carry on from here",
1768                self.state.status.as_str()
1769            ),
1770            None => format!(
1771                "parked after `{}` — resume to carry on from here",
1772                self.state.status.as_str()
1773            ),
1774        };
1775        self.state.event("park", why);
1776        self.state
1777            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1778        self.state.parked = true;
1779        self.state.save()?;
1780        Ok(true)
1781    }
1782
1783    /// Only a new explicit operator approval may retry failed preparation.
1784    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1785        if self.state.prep_failure.is_none() {
1786            return Ok(());
1787        }
1788        let failed_at = self
1789            .state
1790            .events
1791            .iter()
1792            .rev()
1793            .find(|event| event.node == "prep")
1794            .map_or(self.state.updated_at, |event| event.at);
1795        anyhow::ensure!(
1796            approval_at > failed_at,
1797            "preparation retry requires a new explicit release after the failure"
1798        );
1799        self.state.prep_failure = None;
1800        self.state.prep_complete = Some(false);
1801        self.state
1802            .event("prep-retry", "operator approved retry of preparation");
1803        Ok(())
1804    }
1805
1806    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1807    pub fn on_pause(&mut self, pause: Pause) {
1808        self.pause = pause;
1809    }
1810
1811    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1812    /// scheduler asking this one run - and no other - to park so a task
1813    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1814    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1815    /// handle.
1816    pub fn watch_interrupt(&mut self, pause: Pause) {
1817        self.interrupt = pause;
1818    }
1819
1820    /// Abandon this run's own open questions, once `status` has actually
1821    /// settled rather than merely paused.
1822    ///
1823    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1824    /// either back up with the candidates, the review round and the seat
1825    /// sessions already on disk, so a question an implementer asked mid-round
1826    /// may still get a real answer read by a real resume. Only the statuses
1827    /// `resumable` excludes are actually final: the run merged, it reached
1828    /// `Ready` with nothing left to do, it failed outright with no
1829    /// established point to continue from, or every candidate agreed, with
1830    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1831    /// every one of those the seat that asked is gone for good, exactly like
1832    /// the run being deleted under `magi run rm` - so the same cleanup
1833    /// applies, worded for what actually happened instead of "the run was
1834    /// deleted".
1835    ///
1836    /// Best-effort and silent on success: called from every place `status`
1837    /// can land on one of those three, including ones a resumed run revisits,
1838    /// so it must cost nothing when there was nothing open to begin with.
1839    fn settle_questions(&mut self) {
1840        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1841            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1842        }
1843    }
1844
1845    /// The tail of the graph after a trustworthy tally: fold losers, review,
1846    /// gate, merge, and persist.
1847    async fn finish_after_tally(&mut self) -> Result<()> {
1848        // Before the losers are folded: their branches are what the graft
1849        // reads from.
1850        self.graft().await?;
1851        if self.resource_halt()? {
1852            return Ok(());
1853        }
1854        self.fold_losers().await?;
1855        // Before review starts, and again right before the gate: a run's
1856        // review rounds can themselves take long enough for the base to move
1857        // a second time, and the gate is the one node whose "green" gets
1858        // acted on.
1859        self.sync_to_base().await?;
1860        if self.hold_base_sync().await? || self.resource_halt()? {
1861            return Ok(());
1862        }
1863        if self.state.status == RunStatus::AlreadyInBase {
1864            return Ok(());
1865        }
1866        self.review_loop().await?;
1867        if self.resource_halt()? {
1868            return Ok(());
1869        }
1870        self.sync_to_base().await?;
1871        if self.hold_base_sync().await? || self.resource_halt()? {
1872            return Ok(());
1873        }
1874        if self.state.status == RunStatus::AlreadyInBase {
1875            return Ok(());
1876        }
1877        self.gate().await?;
1878        self.merge().await?;
1879        if self.resource_halt()? {
1880            return Ok(());
1881        }
1882        self.state.save()?;
1883        Ok(())
1884    }
1885
1886    // ---------------------------------------------------------------- prep
1887
1888    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1889    /// run with the step and its output in the event; it is never swallowed.
1890    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1891    /// resume starts over instead of finding candidates and missing seats.
1892    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1893        let result = match ensure_setup_config(&mut self.state, repo).await {
1894            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1895            Err(e) => Err(e),
1896        };
1897        if let Err(e) = result {
1898            git::worktree_remove_clean(repo, wt).await.ok();
1899            if in_prep {
1900                self.rollback_prep(repo).await;
1901            }
1902            self.state.status = RunStatus::Blocked;
1903            self.state.event(
1904                "setup",
1905                format!("worktree setup failed in {}: {e:#}", wt.display()),
1906            );
1907            self.state.save()?;
1908            return Err(e);
1909        }
1910        Ok(())
1911    }
1912
1913    /// Undo everything `prep` created, so a resume re-runs it whole.
1914    async fn rollback_prep(&mut self, repo: &Path) {
1915        let root = self.state.worktree_root();
1916        for c in std::mem::take(&mut self.state.candidates) {
1917            if matches!(
1918                git::worktree_remove_clean(repo, &c.worktree).await,
1919                Ok(true)
1920            ) {
1921                git::branch_delete(repo, &c.branch).await.ok();
1922            } else {
1923                self.state.candidates.push(c);
1924            }
1925        }
1926        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1927        for j in 1..=seats {
1928            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1929                .await
1930                .ok();
1931        }
1932        for k in 1..=self.state.config.graph.advisors {
1933            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1934                .await
1935                .ok();
1936        }
1937    }
1938
1939    async fn prep(&mut self) -> Result<()> {
1940        if let Some(reason) = &self.state.prep_failure {
1941            bail!(
1942                "candidate preparation previously failed: {reason}; correct the source before retrying"
1943            );
1944        }
1945        if self
1946            .state
1947            .prep_complete
1948            .unwrap_or(!self.state.candidates.is_empty())
1949        {
1950            return Ok(());
1951        }
1952        let repo = self.state.repo.clone();
1953        // Includes rollback, config, refs, checkout creation and setup. No
1954        // repository mutation may precede the real shared lock acquisition.
1955        let _coordination = self.coordinate(&repo).await?;
1956        let base = self.state.base_commit.clone();
1957        let plan = self.state.sources.plan(&repo, &base).await?;
1958        self.rollback_prep(&repo).await;
1959        if !self.state.candidates.is_empty() {
1960            bail!(
1961                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1962            );
1963        }
1964        self.state.prep_complete = Some(false);
1965        self.state.status = RunStatus::Prep;
1966        self.state.save()?;
1967        let repo = self.state.repo.clone();
1968        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1969        let root = self.state.worktree_root();
1970        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1971
1972        // The hook is the write-time half of the blindness contract; the
1973        // presentation filter in `blind` is the half that cannot be bypassed.
1974        let hooks_dir = self.state.dir().join("hooks");
1975        if self.state.config.blind.commit_msg_hook {
1976            std::fs::create_dir_all(&hooks_dir)
1977                .with_context(|| format!("create {}", hooks_dir.display()))?;
1978            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1979            let path = hooks_dir.join("commit-msg");
1980            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1981            make_executable(&path)?;
1982            // Ref-counted rather than a plain idempotent set: with more than
1983            // one run able to be in flight in the same repository at once
1984            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1985            // true?" check cannot tell "another run of mine still needs
1986            // this" from "nobody does", and the run that happens to finish
1987            // first would disable the hook out from under a sibling still
1988            // relying on it.
1989            git::acquire_worktree_config(&repo).await?;
1990            self.state.enabled_worktree_config = true;
1991        }
1992
1993        // `[worktree] setup` hides its products per worktree, which needs the
1994        // same per-worktree config the hook does. Held until fold, like it.
1995        ensure_setup_config(&mut self.state, &repo).await?;
1996
1997        for (index, (spec, label)) in self
1998            .roles
1999            .implementers
2000            .clone()
2001            .into_iter()
2002            .zip(labels)
2003            .enumerate()
2004        {
2005            let branch = self.state.branch_for(label);
2006            let worktree = root.join(format!("cand-{label}"));
2007            // Persist intent before Git can leave a partially created tree.
2008            // Candidate registration follows setup, so it cannot inventory
2009            // crashes or refusals during creation itself.
2010            self.state.event(
2011                "prep-intent",
2012                serde_json::json!({
2013                    "branch": branch,
2014                    "worktree": worktree,
2015                    "start": start,
2016                    "picks": plan.picks,
2017                })
2018                .to_string(),
2019            );
2020            self.state.save()?;
2021            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2022            if self.state.config.blind.commit_msg_hook {
2023                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2024            }
2025            git::local_exclude(&worktree, "/.magi/").await?;
2026            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2027                // Nothing of this prep may survive: a resume must start over,
2028                // not find half the candidates and skip the rest.
2029                git::worktree_remove_clean(&repo, &worktree).await.ok();
2030                git::branch_delete(&repo, &branch).await.ok();
2031                self.rollback_prep(&repo).await;
2032                self.state.status = RunStatus::Blocked;
2033                self.state
2034                    .event("prep", format!("worktree setup failed: {e:#}"));
2035                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2036                self.state.save()?;
2037                return Err(e);
2038            }
2039            self.state.candidates.push(Candidate {
2040                index,
2041                label,
2042                agent: spec.id.clone(),
2043                branch,
2044                worktree,
2045                summary: String::new(),
2046                stat: String::new(),
2047                files: 0,
2048                commits: 0,
2049                empty: false,
2050                failed: None,
2051                verified_noop: None,
2052                duration_ms: 0,
2053                folded: false,
2054            });
2055            self.state.save()?;
2056            for pick in &plan.picks {
2057                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2058                {
2059                    self.state.status = RunStatus::Blocked;
2060                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2061                    self.state
2062                        .event("prep", format!("cannot apply referenced commit: {e}"));
2063                    self.state.save()?;
2064                    return Err(e);
2065                }
2066            }
2067        }
2068
2069        for j in 1..=self.roles.judges.len() {
2070            let wt = root.join(format!("judge-{j}"));
2071            if !wt.exists() {
2072                git::worktree_add_detached(&repo, &wt, &base).await?;
2073                self.setup_seat_worktree(&repo, &wt, true).await?;
2074            }
2075        }
2076
2077        // Disposable, detached checkouts for the design-deliberation stage's
2078        // advisor seats — the same shape as the judges' above, at the same
2079        // base commit, since advisors also only ever read. Sized off the
2080        // configured count directly rather than a resolved roster: unlike
2081        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2082        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2083        // `prep` has no `ResolvedRoles` field to read a count from here.
2084        if self.state.config.graph.advise {
2085            for k in 1..=self.state.config.graph.advisors {
2086                let wt = root.join(format!("advisor-{k}"));
2087                if !wt.exists() {
2088                    git::worktree_add_detached(&repo, &wt, &base).await?;
2089                    self.setup_seat_worktree(&repo, &wt, true).await?;
2090                }
2091            }
2092        }
2093
2094        // A judge cannot tell it is looking at its own patch — the seats keep
2095        // separate conversations — but a panel that shares agents with the
2096        // field is less independent than it looks, and that is worth saying out
2097        // loud once per run rather than leaving it in the config.
2098        let authors: Vec<&str> = self
2099            .roles
2100            .implementers
2101            .iter()
2102            .map(|a| a.id.as_str())
2103            .collect();
2104        let overlap: Vec<String> = self
2105            .roles
2106            .judges
2107            .iter()
2108            .enumerate()
2109            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2110            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2111            .collect();
2112        if !overlap.is_empty() {
2113            let note = format!(
2114                "{} also authored a candidate; blind, but the panel is less \
2115                 independent than {} distinct agents would be",
2116                overlap.join(", "),
2117                self.roles.judges.len()
2118            );
2119            self.state.event("prep", note);
2120        }
2121
2122        self.state.event(
2123            "prep",
2124            format!(
2125                "{} candidates, {} judges, base {} ({})",
2126                self.state.candidates.len(),
2127                self.roles.judges.len(),
2128                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2129                self.state.base_branch
2130            ),
2131        );
2132        self.state.prep_complete = Some(true);
2133        self.state.status = RunStatus::Implementing;
2134        self.state.save()?;
2135        Ok(())
2136    }
2137
2138    // -------------------------------------------------------------- advise
2139
2140    /// The design-deliberation stage: independent, read-only advisor seats
2141    /// each sketch a design before any implementer touches the repository,
2142    /// and (when at least one produced a usable proposal) a synthesis seat
2143    /// blends them into a brief `implement` carries in every candidate's
2144    /// prompt.
2145    ///
2146    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2147    /// advisors` is the proposal count. Everything here is best-effort and
2148    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2149    /// that cannot reach quota, or a synthesis seat that produced nothing
2150    /// usable all leave `implement` exactly as it was before this stage
2151    /// existed — the task instruction alone — rather than failing the whole
2152    /// competition over an enrichment stage. Every outcome is still recorded
2153    /// as an event, so a run that got nothing from this stage says why.
2154    ///
2155    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2156    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2157    /// resumed run whose stage failed would re-run it, and re-spend the
2158    /// agent calls, on every reentry before `implement`.
2159    ///
2160    /// Also skipped once any candidate shows implementation progress — the
2161    /// exact predicate `implement` itself uses to decide a candidate is no
2162    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2163    /// is not enough: a run created by an older binary that predates this
2164    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2165    /// an already-`Implementing`-or-later run under this build would
2166    /// otherwise walk straight back through `prep` (a no-op once candidates
2167    /// exist) into this node and spawn every advisor seat against worktrees
2168    /// `prep` never recreated — after implementation has already started,
2169    /// which is exactly the invariant this stage exists to guarantee.
2170    async fn advise(&mut self) -> Result<()> {
2171        let implement_untouched = self
2172            .state
2173            .candidates
2174            .iter()
2175            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2176        if !self.state.config.graph.advise || self.state.advise_attempted {
2177            return Ok(());
2178        }
2179        if !implement_untouched {
2180            self.state.event(
2181                "advise",
2182                "skipping the design-deliberation stage: at least one \
2183                 candidate already shows implementation progress, so this \
2184                 run is past the point the stage exists to run before"
2185                    .to_owned(),
2186            );
2187            self.state.advise_attempted = true;
2188            self.state.save()?;
2189            return Ok(());
2190        }
2191        let run_id = self.state.id.clone();
2192        let prompts = self.state.config.prompts.clone();
2193        let instruction = self.state.instruction.clone();
2194        let language = self.state.config.graph.language.clone();
2195        let root = self.state.worktree_root();
2196        let n = self.state.config.graph.advisors;
2197        let where_recorded = self.state.dir().join("run.json");
2198
2199        let seats = match self.state.config.advisors() {
2200            Ok(seats) if !seats.is_empty() => seats,
2201            Ok(_) => {
2202                self.state.event(
2203                    "advise",
2204                    format!(
2205                        "[graph] advisors is 0; skipping the design-deliberation \
2206                         stage and continuing without a synthesis brief (see {})",
2207                        where_recorded.display()
2208                    ),
2209                );
2210                self.state.advise_attempted = true;
2211                self.state.save()?;
2212                return Ok(());
2213            }
2214            Err(e) => {
2215                self.state.event(
2216                    "advise",
2217                    format!(
2218                        "could not resolve advisor seats ({e:#}); continuing \
2219                         without a design-deliberation brief (see {})",
2220                        where_recorded.display()
2221                    ),
2222                );
2223                self.state.advise_attempted = true;
2224                self.state.save()?;
2225                return Ok(());
2226            }
2227        };
2228
2229        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2230        let artifacts = agent::artifacts_dir(&self.state.dir());
2231        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2232
2233        let mut jobs = Vec::new();
2234        for (i, spec) in seats.iter().cloned().enumerate() {
2235            let seat_key = format!("advisor-{}", i + 1);
2236            let seat = self.seat(&seat_key, &spec.id);
2237            jobs.push(SeatJob {
2238                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2239                spec,
2240                seat,
2241                cwd: worktrees[i % worktrees.len()].clone(),
2242                timeout,
2243                allow_write: false,
2244                sessions: false,
2245                artifacts: artifacts.clone(),
2246                stem: seat_key,
2247                handover: None,
2248            });
2249        }
2250
2251        self.state.event(
2252            "advise",
2253            format!(
2254                "{} advisor seat(s) sketching a design in parallel",
2255                jobs.len()
2256            ),
2257        );
2258        let mut quota_losses = Vec::new();
2259        let cache = self.state.config.cache_dir();
2260        let ctx = WaveCtx {
2261            carry_seats: false,
2262            pauses: Some((&self.pause, &self.interrupt)),
2263            run: &run_id,
2264            node: "advise",
2265            prompts: &prompts,
2266            cache: cache.as_deref(),
2267            round: None,
2268        };
2269        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2270        let results = ask_json_wave::<Proposal>(
2271            jobs,
2272            Arc::clone(&self.sem),
2273            self.state.config.graph.retries,
2274            &advisor_roster,
2275            &ctx,
2276            &mut quota_losses,
2277            &mut self.state,
2278            &|p: &Proposal| p.validate(),
2279        )
2280        .await;
2281        if self.state.resource_block.is_some() {
2282            return self.stop_for_resource();
2283        }
2284        self.state.quota.extend(quota_losses);
2285
2286        let mut records = Vec::with_capacity(results.len());
2287        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2288            let agent_id = seat.agent.clone();
2289            self.state.seats.insert(seat.key.clone(), seat);
2290            match res {
2291                Ok((proposal, out)) => {
2292                    self.state
2293                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2294                    records.push(advise::AdvisorRecord::proposed(
2295                        i + 1,
2296                        agent_id,
2297                        proposal,
2298                        out.duration_ms,
2299                    ));
2300                }
2301                Err(e) => {
2302                    self.state.event(
2303                        "advise",
2304                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2305                    );
2306                    records.push(advise::AdvisorRecord::failed(
2307                        i + 1,
2308                        agent_id,
2309                        e.to_string(),
2310                    ));
2311                }
2312            }
2313        }
2314
2315        let mut advice = advise::Advice {
2316            records,
2317            synthesis: None,
2318        };
2319        if advice.proposals().is_empty() {
2320            self.state.event(
2321                "advise",
2322                "no advisor produced a usable proposal; continuing without a \
2323                 synthesis brief"
2324                    .to_owned(),
2325            );
2326        } else {
2327            match self
2328                .synthesize_brief(
2329                    &advice,
2330                    &instruction,
2331                    &language,
2332                    &worktrees[0],
2333                    &artifacts,
2334                    &run_id,
2335                    &prompts,
2336                    cache.as_deref(),
2337                )
2338                .await
2339            {
2340                Ok(Some(text)) => {
2341                    self.state.event(
2342                        "advise",
2343                        "synthesized a design brief for the implementer".to_owned(),
2344                    );
2345                    advice.synthesis = Some(text);
2346                }
2347                Ok(None) => {
2348                    self.state.event(
2349                        "advise",
2350                        "the synthesis seat produced nothing usable; continuing \
2351                         without a design brief"
2352                            .to_owned(),
2353                    );
2354                }
2355                Err(e) => {
2356                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2357                        return Err(e);
2358                    }
2359                    self.state.event(
2360                        "advise",
2361                        format!("could not synthesize a design brief: {e:#}"),
2362                    );
2363                }
2364            }
2365        }
2366        advise::apply_reflection(&mut advice);
2367
2368        self.state.advice = Some(advice);
2369        self.state.advise_attempted = true;
2370        self.state.save()?;
2371        Ok(())
2372    }
2373
2374    /// The synthesis seat: reads every advisor's proposal and blends them
2375    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2376    /// out of [`Runner::advise`] only for readability — it is not called
2377    /// anywhere else.
2378    ///
2379    /// Picked the same way [`crate::talk`]'s standing conversation and
2380    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2381    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2382    /// order (a claude seat, else the first runnable agent in roster order)
2383    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2384    /// in [`crate::config`] for why a dedicated field exists here at all.
2385    #[allow(clippy::too_many_arguments)]
2386    async fn synthesize_brief(
2387        &mut self,
2388        advice: &advise::Advice,
2389        instruction: &str,
2390        language: &str,
2391        cwd: &Path,
2392        artifacts: &Path,
2393        run_id: &str,
2394        prompts: &Prompts,
2395        cache: Option<&Path>,
2396    ) -> Result<Option<String>> {
2397        let chain = agent::pick_chain(
2398            &self.state.config.agents,
2399            self.state.config.roles.synthesizer.as_ref(),
2400            &agent::installed,
2401            "synthesizer",
2402        )?;
2403        let proposals = advice.proposals();
2404        let mut prompt = prompt::with_overlay(
2405            prompt::synthesize_brief(instruction, &proposals, language),
2406            prompts.overlay("advise"),
2407        );
2408        if cache.is_some() {
2409            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2410            // below — see `prompt::build_cache_note`'s doc for why telling a
2411            // read-only seat to build through the shared cache is exactly how
2412            // a sandbox's write refusal gets misread as a defect.
2413            prompt.push('\n');
2414            prompt.push_str(&prompt::build_cache_note("advise", false));
2415        }
2416        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2417        // Each id is tried once, in order; a quota hit, error or unusable
2418        // answer moves to the next. The seat is single-turn (`sessions:
2419        // false`) and the prompt is the whole context, so a fallback agent
2420        // needs nothing carried over.
2421        let mut last = None;
2422        for (n, spec) in chain.iter().enumerate() {
2423            if n > 0 {
2424                self.state
2425                    .event("advise", format!("synthesis falling back to {}", spec.id));
2426            }
2427            let mut seat = self.seat("advise-synthesis", &spec.id);
2428            let wait = self.repository_wait();
2429            let outcome = wait
2430                .scope(agent::invoke(
2431                    spec,
2432                    &mut seat,
2433                    &Invocation {
2434                        coordination: self.state.config.coordination.as_ref(),
2435                        fetched_base: None,
2436                        cwd,
2437                        prompt: &prompt,
2438                        timeout,
2439                        allow_write: false,
2440                        sessions: false,
2441                        artifacts,
2442                        stem: &if n == 0 {
2443                            "advise-synthesis".to_owned()
2444                        } else {
2445                            format!("advise-synthesis-{}", spec.id)
2446                        },
2447                        run: run_id,
2448                        node: "advise",
2449                        cache_dir: None,
2450                        attachments: &[],
2451                        writable: &[],
2452                    },
2453                ))
2454                .await;
2455            if outcome.is_ok() {
2456                self.state.seats.insert(seat.key.clone(), seat);
2457            }
2458            let advance = agent::chain_advances(&outcome);
2459            last = Some(outcome);
2460            if !advance {
2461                break;
2462            }
2463        }
2464        // Exhausted: the last attempt's result is what a single failed seat
2465        // would have produced.
2466        let out = last.expect("a chain holds at least one agent")?;
2467        if !out.usable() {
2468            return Ok(None);
2469        }
2470        let text =
2471            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2472        Ok((!text.trim().is_empty()).then_some(text))
2473    }
2474
2475    // ----------------------------------------------------------- implement
2476
2477    async fn implement(&mut self) -> Result<()> {
2478        self.persist_candidates().await?;
2479        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2480        // agent files with `magi task add` name the run that paid for it. The
2481        // prompt overlay is cloned alongside it because the waves borrow it
2482        // while `self` is mutably borrowed by the node's own bookkeeping.
2483        let run_id = self.state.id.clone();
2484        let prompts = self.state.config.prompts.clone();
2485        let todo: Vec<usize> = self
2486            .state
2487            .candidates
2488            .iter()
2489            .enumerate()
2490            .filter(|(_, c)| {
2491                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2492                    && c.failed.is_none()
2493                    && !c.empty
2494            })
2495            .map(|(i, _)| i)
2496            .collect();
2497        if todo.is_empty() {
2498            return self.after_implement();
2499        }
2500        self.state.status = RunStatus::Implementing;
2501
2502        let language = self.state.config.graph.language.clone();
2503        let github_language = self.state.config.graph.github_language.clone();
2504        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2505        let sessions = self.state.config.graph.sessions;
2506        let artifacts = agent::artifacts_dir(&self.state.dir());
2507        // The design-deliberation stage's blended brief, when `advise` found
2508        // one — carried into every implementer's prompt the same way
2509        // regardless of which candidate it is.
2510        let brief = self
2511            .state
2512            .advice
2513            .as_ref()
2514            .and_then(|a| a.synthesis.as_deref())
2515            .map(str::to_owned);
2516        let attachments = self.state.attachments.clone();
2517
2518        let mut jobs = Vec::new();
2519        for &i in &todo {
2520            let (index, label, worktree) = {
2521                let c = &self.state.candidates[i];
2522                (c.index, c.label, c.worktree.clone())
2523            };
2524            let spec = self.roles.implementers[index].clone();
2525            let seat_key = format!("impl-{label}");
2526            let seat = self.seat(&seat_key, &spec.id);
2527            let instruction = seeded_instruction(&self.state);
2528            jobs.push(SeatJob {
2529                spec,
2530                seat,
2531                prompt: prompt::implement_in(
2532                    &instruction,
2533                    &worktree.to_string_lossy(),
2534                    &language,
2535                    &github_language,
2536                    brief.as_deref(),
2537                    &attachments,
2538                ),
2539                cwd: worktree,
2540                timeout,
2541                allow_write: true,
2542                sessions,
2543                artifacts: artifacts.clone(),
2544                stem: format!("impl-{label}"),
2545                handover: None,
2546            });
2547        }
2548
2549        self.state.event(
2550            "implement",
2551            format!("{} candidates in parallel", jobs.len()),
2552        );
2553        // Kept so a seat whose CLI hung up can be asked again from the same
2554        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2555        // can update a seat's own entry once a fallback agent takes it over —
2556        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2557        // whichever agent actually answered, not the one that quota'd out.
2558        let mut sent = jobs.clone();
2559        let cache = self.state.config.cache_dir();
2560        let ctx = WaveCtx {
2561            carry_seats: false,
2562            pauses: Some((&self.pause, &self.interrupt)),
2563            run: &run_id,
2564            node: "implement",
2565            prompts: &prompts,
2566            cache: cache.as_deref(),
2567            round: None,
2568        };
2569        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2570        if !wave_blocked(&results) {
2571            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2572                .await;
2573            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2574                .await?;
2575            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2576                .await;
2577        }
2578        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2579
2580        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2581            && results
2582                .iter()
2583                .any(|(_, _, out)| FailClass::of(out).is_some());
2584        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2585            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2586                // Executed turns interrupted by disk pressure already carry
2587                // a preservation barrier from wave. Admission-only waits do
2588                // not. Keep the interrupted seat's session for recovery too.
2589                if self
2590                    .state
2591                    .candidate_preservation
2592                    .contains_key(&self.state.candidates[i].label)
2593                {
2594                    self.state.seats.insert(seat.key.clone(), seat);
2595                }
2596                continue;
2597            }
2598            let seat_key = seat.key.clone();
2599            // A quota fallback (`resume_seat_handovers`) may have handed this
2600            // seat to a different agent than the one `prep` recorded on the
2601            // candidate; the stats tables and any later fixer-defaults-to-
2602            // winner's-author lookup must credit whoever actually answered —
2603            // unless every fallback also quota'd out, in which case nobody
2604            // actually answered and crediting the last agent tried would
2605            // erase every earlier agent's own quota loss from the stats
2606            // tables instead of just this one seat's.
2607            let agent = seat.agent.clone();
2608            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2609            self.state.seats.insert(seat.key.clone(), seat);
2610            let label = self.state.candidates[i].label;
2611            self.state.interrupted_implementations.remove(&label);
2612
2613            let (summary, duration, failed, verified_claim) = match out {
2614                AgentOutcome::Ok(o) => {
2615                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2616                    let failed = (!o.usable()).then(|| {
2617                        if o.timed_out {
2618                            "agent timed out".to_owned()
2619                        } else {
2620                            format!(
2621                                "agent exited with {}",
2622                                crate::notices::exit_status(o.exit_code)
2623                            )
2624                        }
2625                    });
2626                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2627                    (text, o.duration_ms, failed, verified_claim)
2628                }
2629                // Left un-resumed by `resume_undelivered` (a dirty tree
2630                // already rescues the work, or there was no session left to
2631                // resume into) — reported like the ordinary failure it is,
2632                // never as if `o.text` (the CLI's raw error JSON) were an
2633                // answer.
2634                AgentOutcome::Dropped(o) => {
2635                    let why = o
2636                        .dropped
2637                        .as_ref()
2638                        .map(|d| d.why.as_str())
2639                        .unwrap_or("the CLI ended the stream without delivering its answer");
2640                    (
2641                        String::new(),
2642                        o.duration_ms,
2643                        Some(format!("the CLI dropped the stream ({why})")),
2644                        None,
2645                    )
2646                }
2647                AgentOutcome::Quota(o) => {
2648                    self.state.quota.push(QuotaLoss {
2649                        seat: seat_key,
2650                        node: "implement".to_owned(),
2651                        at: Timestamp::now(),
2652                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2653                    });
2654                    (
2655                        String::new(),
2656                        o.duration_ms,
2657                        Some("rate limited (quota); produced no change".to_owned()),
2658                        None,
2659                    )
2660                }
2661                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2662                    (String::new(), 0, Some(e), None)
2663                }
2664            };
2665
2666            // Save every completed turn before any fallible Git operation. A failed
2667            // rescue must not erase the reply or prevent collection of other seats.
2668            let c = &mut self.state.candidates[i];
2669            if !exhausted_the_fallback_chain {
2670                c.agent = agent;
2671            }
2672            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2673            c.duration_ms = duration;
2674            c.failed = failed;
2675            c.verified_noop = verified_claim;
2676            self.state
2677                .candidate_preservation
2678                .insert(label, "awaiting persistence".to_owned());
2679        }
2680        self.state.save()?;
2681        self.persist_candidates().await?;
2682
2683        if resource_blocked {
2684            return self.stop_for_resource();
2685        }
2686        if selected_failed {
2687            self.state.status = RunStatus::Blocked;
2688            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2689            self.state.save()?;
2690            bail!(
2691                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2692            );
2693        }
2694        self.after_implement()
2695    }
2696
2697    /// Retry only persistence after a completed implementation turn.
2698    async fn persist_candidates(&mut self) -> Result<()> {
2699        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2700        for label in labels {
2701            let i = self
2702                .state
2703                .candidates
2704                .iter()
2705                .position(|c| c.label == label)
2706                .context("pending candidate is missing")?;
2707            let c = self.state.candidates[i].clone();
2708            let interrupted = self.state.interrupted_implementations.contains(&label);
2709            let result = async {
2710                let wait = crate::repo_lock::WaitPolicy::new(
2711                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2712                    || false,
2713                );
2714                let _coordination =
2715                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2716                // Best effort evidence requires no index writes. A successful snapshot
2717                // never authorizes removal of a tree whose rescue failed.
2718                if let Err(e) = git::snapshot_uncommitted(
2719                    &c.worktree,
2720                    &self
2721                        .state
2722                        .dir()
2723                        .join("artifacts")
2724                        .join(format!("candidate-{label}-recovery")),
2725                )
2726                .await
2727                {
2728                    self.state.event(
2729                        "preserve",
2730                        format!("candidate {label} snapshot incomplete: {e:#}"),
2731                    );
2732                }
2733                let rescued = git::rescue_commit(
2734                    &c.worktree,
2735                    &format!("magi: candidate {label} (uncommitted work)"),
2736                )
2737                .await?;
2738                self.state.note_withheld("implement", &rescued.withheld);
2739                let base = &self.state.base_commit;
2740                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2741                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2742                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2743                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2744                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2745                if c.empty && commits > 0 && !patch.trim().is_empty()
2746                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2747                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2748                        || self.state.tally.is_some())
2749                {
2750                    // Keep the old classification and pending marker so every
2751                    // retry stops too. Retain decisions as recovery evidence;
2752                    // only a supported recovery operation may replace them.
2753                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2754                }
2755                let c = &mut self.state.candidates[i];
2756                c.commits = commits;
2757                c.stat = stat;
2758                c.files = files;
2759                let no_changes = commits == 0 || patch.trim().is_empty();
2760                c.empty = no_changes && !interrupted;
2761                if interrupted && no_changes {
2762                    c.failed = None;
2763                    c.verified_noop = None;
2764                }
2765                if !c.empty {
2766                    c.failed = None;
2767                    c.verified_noop = None;
2768                }
2769                self.state.event(
2770                    "preserve",
2771                    format!("candidate {label}: {files} files, {commits} commits"),
2772                );
2773                Ok::<_, anyhow::Error>(())
2774            }
2775            .await;
2776            match result {
2777                Ok(()) => {
2778                    self.state.candidate_preservation.remove(&label);
2779                    if !interrupted || self.state.candidates[i].files > 0 {
2780                        self.state.interrupted_implementations.remove(&label);
2781                    }
2782                }
2783                Err(e) => {
2784                    let reason = format!("candidate {label} preservation failed: {e:#}");
2785                    self.state.event("preserve", &reason);
2786                    self.state.candidate_preservation.insert(label, reason);
2787                }
2788            }
2789            if self.state.candidate_preservation.is_empty()
2790                && self.state.status == RunStatus::Blocked
2791                && self.state.candidates.iter().all(|c| c.failed.is_none())
2792            {
2793                // Persist the recovered phase with the cleared pending marker;
2794                // a crash between saves must not turn this into a selected-seat
2795                // failure hold on the next resume.
2796                self.state.status = RunStatus::Implementing;
2797                self.state.event(
2798                    "preserve",
2799                    "candidate persistence recovered; implementation can continue",
2800                );
2801            }
2802            self.state.save()?;
2803        }
2804        if !self.state.candidate_preservation.is_empty() {
2805            self.state.status = RunStatus::Blocked;
2806            self.state.save()?;
2807            bail!(
2808                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2809            );
2810        }
2811        Ok(())
2812    }
2813
2814    /// Ask again, once, for work a CLI did and then failed to hand over.
2815    ///
2816    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2817    /// status with an empty response and a usage report showing output tokens,
2818    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2819    /// seven minutes and 14,267 output tokens that arrived as an empty
2820    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2821    ///
2822    /// Two conditions, and both matter:
2823    ///
2824    /// - **Only when the tree is untouched.** Often the agent has already
2825    ///   written its files and only the closing message was lost; the rescue
2826    ///   commit below picks that up and there is nothing to ask for. Re-asking
2827    ///   then would pay for a second implementation of work already on disk.
2828    /// - **Once.** A CLI that drops one stream can drop the next, and this
2829    ///   node is the most expensive in the graph.
2830    ///
2831    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2832    /// dropped reply still carried its `conversation_id`, so the seat is asked
2833    /// to finish what it was doing rather than sent the whole task again. It
2834    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2835    /// node's - for the same reason a re-ranked judge does: restating finished
2836    /// work is not the work.
2837    ///
2838    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2839    /// same way until it resets, while an abandoned conversation is still
2840    /// there to be picked up.
2841    async fn resume_undelivered(
2842        &mut self,
2843        results: &mut [(usize, SeatState, AgentOutcome)],
2844        sent: &[SeatJob],
2845        prompts: &Prompts,
2846        run_id: &str,
2847    ) {
2848        for (wi, seat, out) in results.iter_mut() {
2849            let Some(dropped) = (match &*out {
2850                AgentOutcome::Dropped(o) => o.dropped.clone(),
2851                _ => None,
2852            }) else {
2853                continue;
2854            };
2855            let Some(job) = sent.get(*wi) else { continue };
2856            // Already on disk? Then only the closing message was lost.
2857            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2858                self.state.event(
2859                    "implement",
2860                    format!(
2861                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2862                         work is in the tree",
2863                        seat.key, dropped.output_tokens, dropped.why
2864                    ),
2865                );
2866                continue;
2867            }
2868            // The re-ask only makes sense as a resume: `resume_after_drop`
2869            // says nothing about the task, trusting the seat to still hold it.
2870            // Without a session to resume — sessions disabled, or this CLI's
2871            // drop shape happened not to carry a session id — that prompt
2872            // would open a brand-new conversation with no context at all,
2873            // which is worse than leaving this as the ordinary failure it
2874            // already is.
2875            if !has_context(&job.spec, seat, job.sessions) {
2876                self.state.event(
2877                    "implement",
2878                    format!(
2879                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2880                         is no session left to resume",
2881                        seat.key, dropped.output_tokens, dropped.why
2882                    ),
2883                );
2884                continue;
2885            }
2886            self.state.event(
2887                "implement",
2888                format!(
2889                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2890                     conversation",
2891                    seat.key, dropped.output_tokens, dropped.why
2892                ),
2893            );
2894            let mut retry = job.clone();
2895            retry.seat = seat.clone();
2896            retry.prompt = prompt::resume_after_drop(&dropped.why);
2897            retry.timeout = retry_budget(job.timeout, true);
2898            retry.stem = format!("{}-resume", job.stem);
2899            let cache = self.state.config.cache_dir();
2900            let ctx = WaveCtx {
2901                carry_seats: false,
2902                pauses: Some((&self.pause, &self.interrupt)),
2903                run: run_id,
2904                node: "implement",
2905                prompts,
2906                cache: cache.as_deref(),
2907                round: None,
2908            };
2909            let (resumed_seat, resumed) =
2910                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2911            *seat = resumed_seat;
2912            *out = resumed;
2913        }
2914    }
2915
2916    /// Fall an implement seat through to the next untried agent in the
2917    /// implementer roster when it lost to quota — or, since the handover was
2918    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2919    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2920    /// quota path itself being unchanged — instead of leaving the
2921    /// seat's loss final the moment one agent's account runs dry.
2922    ///
2923    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2924    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2925    /// truncates to the single slot rotation picked, so a solo task whose one
2926    /// implementer hits quota mid-run used to have nothing else to try. This
2927    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2928    /// unrotated roster — which is the only place the *other* candidates in
2929    /// the machine's roster still exist once `implementers` has been cut down
2930    /// to size.
2931    ///
2932    /// Walks forward from just past the seat's own original position in the
2933    /// roster, never wrapping back to the front: a later candidate slot (say
2934    /// `beta`, the roster's second entry) must fall through to the *next*
2935    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2936    /// almost certainly a different candidate's own agent already — and once
2937    /// the roster's tail is exhausted there is nothing left to fall through
2938    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2939    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2940    /// let this retry that id forever. The loop keeps falling through until
2941    /// an attempt lands something other than `Quota` or the roster's tail
2942    /// runs out of untried ids, at which point the seat is left exactly as
2943    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2944    /// `QuotaLoss` recorded, the candidate failed/empty.
2945    ///
2946    /// `sent` is taken mutably and updated with the fallback agent's spec:
2947    /// `resume_unconfirmed_commands`, which runs after this and also reads
2948    /// `sent`, must see whichever agent actually ended up answering the seat
2949    /// — reading the stale, original spec there would check session
2950    /// eligibility against the wrong CLI and could hand a fallback agent's
2951    /// session id to the agent that just lost the seat to quota.
2952    ///
2953    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2954    /// — `self.seat` only reuses state when the agent id is unchanged, so
2955    /// handing it a different id already gets this for free. Reusing the old
2956    /// seat would resume a different CLI's session as if it were a
2957    /// continuation of this one.
2958    ///
2959    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2960    /// a quota loss cuts an agent off mid-turn, so anything already in the
2961    /// tree is unfinished work, not a completed candidate a re-ask would pay
2962    /// for twice. A dirty tree is rescued into a commit first (the same
2963    /// neutral-identity rescue `implement`'s own outcome loop gives every
2964    /// candidate) so the next agent starts clean.
2965    ///
2966    /// The new agent gets the implementer's full prompt and full
2967    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2968    /// it has no session and no context, and is implementing the task from
2969    /// nothing, unlike a resumed drop which is only restating work already
2970    /// done.
2971    ///
2972    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2973    /// `implement` event, never into `self.state.quota` — that is what
2974    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2975    /// go unspent, and a seat that ultimately recovered on its second or
2976    /// third agent is not the stalled panel that check exists to catch. Only
2977    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2978    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2979    /// outcome loop already has — this helper never pushes to it itself.
2980    async fn resume_seat_handovers(
2981        &mut self,
2982        results: &mut [(usize, SeatState, AgentOutcome)],
2983        sent: &mut [SeatJob],
2984        prompts: &Prompts,
2985        run_id: &str,
2986    ) -> Result<()> {
2987        let instruction = seeded_instruction(&self.state);
2988        let language = self.state.config.graph.language.clone();
2989        let github_language = self.state.config.graph.github_language.clone();
2990        let brief = self
2991            .state
2992            .advice
2993            .as_ref()
2994            .and_then(|a| a.synthesis.as_deref())
2995            .map(str::to_owned);
2996        let attachments = self.state.attachments.clone();
2997        for (wi, seat, out) in results.iter_mut() {
2998            // Who holds the other candidate seats of this wave right now
2999            // (earlier handovers already written back to `sent`).
3000            let others: BTreeSet<String> = sent
3001                .iter()
3002                .enumerate()
3003                .filter(|(j, _)| j != wi)
3004                .map(|(_, j)| j.spec.id.clone())
3005                .collect();
3006            let Some(job) = sent.get_mut(*wi) else {
3007                continue;
3008            };
3009            // Where the seat's own original agent sits in the roster — the
3010            // fallback walk starts just past here, never at the front, so a
3011            // later candidate slot's quota loss does not fall back onto an
3012            // earlier slot's own agent.
3013            let start = self
3014                .roles
3015                .implementer_roster
3016                .iter()
3017                .position(|s| s.id == job.spec.id)
3018                .unwrap_or(0);
3019            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3020            let mut fallback_attempt = 0usize;
3021            let mut prev: Option<FailClass> = None;
3022            while let Some(cur) = FailClass::of(&*out) {
3023                if !should_hand_over(prev.as_ref(), &cur) {
3024                    break;
3025                }
3026                let Some(next) =
3027                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3028                        .cloned()
3029                else {
3030                    break;
3031                };
3032                tried.insert(next.id.clone());
3033                fallback_attempt += 1;
3034
3035                let coordination = self.coordinate(&job.cwd).await?;
3036                if let Ok(r) = git::rescue_commit(
3037                    &job.cwd,
3038                    &format!(
3039                        "magi: candidate {} (uncommitted work before {} fallback)",
3040                        seat.key,
3041                        if cur == FailClass::Quota {
3042                            "quota"
3043                        } else {
3044                            "handover"
3045                        }
3046                    ),
3047                )
3048                .await
3049                {
3050                    self.state.note_withheld("implement", &r.withheld);
3051                }
3052                drop(coordination);
3053
3054                record_handover(
3055                    &mut self.state,
3056                    "implement",
3057                    &seat.key,
3058                    &seat.agent,
3059                    &next.id,
3060                    &cur,
3061                    &fail_reason(&*out),
3062                );
3063                prev = Some(cur.clone());
3064
3065                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3066                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3067                // Kept in sync on `sent` itself, not just the local retry: a
3068                // later helper (`resume_unconfirmed_commands`) reads `sent`
3069                // after this one returns and must see whichever agent is now
3070                // occupying the seat, not the one that just quota'd out —
3071                // otherwise it would judge session/continuation eligibility
3072                // by the wrong CLI and could resend a fallback's session id
3073                // to the agent that lost it the seat in the first place.
3074                job.spec = next.clone();
3075                let mut retry = job.clone();
3076                retry.seat = new_seat;
3077                retry.prompt = prompt::implement_in(
3078                    &instruction,
3079                    &job.cwd.to_string_lossy(),
3080                    &language,
3081                    &github_language,
3082                    brief.as_deref(),
3083                    &attachments,
3084                );
3085                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3086                let cache = self.state.config.cache_dir();
3087                let ctx = WaveCtx {
3088                    carry_seats: false,
3089                    pauses: Some((&self.pause, &self.interrupt)),
3090                    run: run_id,
3091                    node: "implement",
3092                    prompts,
3093                    cache: cache.as_deref(),
3094                    round: None,
3095                };
3096                let (fallback_seat, fallback_out) = run_one(
3097                    retry,
3098                    Arc::clone(&self.sem),
3099                    &ctx,
3100                    &mut self.state,
3101                    fallback_attempt,
3102                )
3103                .await;
3104                *seat = fallback_seat;
3105                *out = fallback_out;
3106            }
3107        }
3108        Ok(())
3109    }
3110
3111    /// Ask an implement seat's own CLI to confirm what it started, once, when
3112    /// its reply reported a command whose completion status it never
3113    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3114    /// that does and does not mean.
3115    ///
3116    /// The completion contract this task asks for, extended to `implement`
3117    /// with the same signal `continue_fix_report` reads for the fixer,
3118    /// rather than a keyword search over the reply or a hard requirement on
3119    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3120    /// a candidate's CLI turn ended cleanly while a test run it had started
3121    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3122    /// named in it at all is untouched by this: `commands` is empty, so
3123    /// there is nothing to be unconfirmed.
3124    ///
3125    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3126    /// this is not about recovering edits that might already be on disk, it
3127    /// is about a result the seat itself never vouched for, which resuming
3128    /// asks for regardless of what the tree already holds. Bounded to one
3129    /// attempt for the same reason `resume_undelivered` is — this is the
3130    /// most expensive node in the graph — and a seat that still cannot
3131    /// confirm on that attempt is left as whatever its (possibly still
3132    /// unconfirmed) reply says; this does not invent a new "failed" reason
3133    /// for a candidate that otherwise produced a real, committed change.
3134    async fn resume_unconfirmed_commands(
3135        &mut self,
3136        results: &mut [(usize, SeatState, AgentOutcome)],
3137        sent: &[SeatJob],
3138        prompts: &Prompts,
3139        run_id: &str,
3140    ) {
3141        for (wi, seat, out) in results.iter_mut() {
3142            let AgentOutcome::Ok(o) = &*out else {
3143                continue;
3144            };
3145            if !has_unconfirmed_command(&o.commands) {
3146                continue;
3147            }
3148            let Some(job) = sent.get(*wi) else { continue };
3149            if !has_context(&job.spec, seat, job.sessions) {
3150                self.state.event(
3151                    "implement",
3152                    format!(
3153                        "{}: the reply named a command whose own CLI never confirmed the exit \
3154                         status of, but there is no session left to resume",
3155                        seat.key
3156                    ),
3157                );
3158                continue;
3159            }
3160            self.state.event(
3161                "implement",
3162                format!(
3163                    "{}: the reply named a command whose own CLI never confirmed the exit \
3164                     status of; resuming the conversation",
3165                    seat.key
3166                ),
3167            );
3168            let mut retry = job.clone();
3169            retry.seat = seat.clone();
3170            retry.prompt = prompt::resume_incomplete(
3171                "a command in your last reply had no confirmed exit status",
3172            );
3173            retry.timeout = retry_budget(job.timeout, true);
3174            retry.stem = format!("{}-confirm", job.stem);
3175            let cache = self.state.config.cache_dir();
3176            let ctx = WaveCtx {
3177                carry_seats: false,
3178                pauses: Some((&self.pause, &self.interrupt)),
3179                run: run_id,
3180                node: "implement",
3181                prompts,
3182                cache: cache.as_deref(),
3183                round: None,
3184            };
3185            let (resumed_seat, resumed) =
3186                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3187            *seat = resumed_seat;
3188            *out = resumed;
3189        }
3190    }
3191
3192    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3193    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3194    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3195    /// that motivated this.
3196    ///
3197    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3198    /// own nudge loop already covers for judge/review/vote seats, and not a
3199    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3200    /// implement seats: here the CLI turn genuinely finished while the node's
3201    /// own work — the fixer's account of what it did — had not. Gated purely
3202    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3203    /// reply, never on any wording in it, so a fixer whose valid, first-try
3204    /// `FixReport` happens to mention having waited on a background test is
3205    /// never resumed — the `Ok(report)` branch at the call site returns
3206    /// before this is ever invoked.
3207    ///
3208    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3209    /// attempt ([`retry_budget`]), nothing attempted once the session is
3210    /// gone, and a quota hit ends the loop immediately rather than retrying a
3211    /// rate limit that fails the same way again.
3212    async fn continue_fix_report(
3213        &mut self,
3214        mut seat: SeatState,
3215        parse_err: String,
3216        job: &SeatJob,
3217        prompts: &Prompts,
3218        run_id: &str,
3219        round: usize,
3220    ) -> (
3221        SeatState,
3222        Option<FixReport>,
3223        Option<String>,
3224        ContinuationRecord,
3225    ) {
3226        let mut last_err = parse_err;
3227        let mut cumulative_wait_ms = 0u64;
3228        let mut attempts = 0usize;
3229        loop {
3230            if !has_context(&job.spec, &seat, job.sessions) {
3231                self.state.event(
3232                    "fix",
3233                    format!(
3234                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3235                         session left to resume into"
3236                    ),
3237                );
3238                let outcome = if attempts == 0 {
3239                    ContinuationOutcome::NoSession
3240                } else {
3241                    ContinuationOutcome::Exhausted
3242                };
3243                return (
3244                    seat,
3245                    None,
3246                    Some(format!("unparsable fix report: {last_err}")),
3247                    ContinuationRecord {
3248                        attempts,
3249                        cumulative_wait_ms,
3250                        outcome,
3251                    },
3252                );
3253            }
3254            if attempts >= MAX_FIX_CONTINUATIONS {
3255                self.state.event(
3256                    "fix",
3257                    format!(
3258                        "round {round}: fixer's reply still had no adoption report after \
3259                         {attempts} continuation(s) ({last_err}); giving up"
3260                    ),
3261                );
3262                return (
3263                    seat,
3264                    None,
3265                    Some(format!(
3266                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3267                    )),
3268                    ContinuationRecord {
3269                        attempts,
3270                        cumulative_wait_ms,
3271                        outcome: ContinuationOutcome::Exhausted,
3272                    },
3273                );
3274            }
3275            attempts += 1;
3276            self.state.event(
3277                "fix",
3278                format!(
3279                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3280                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3281                ),
3282            );
3283            let mut retry = job.clone();
3284            retry.seat = seat.clone();
3285            retry.prompt = prompt::resume_incomplete(&last_err);
3286            retry.timeout = retry_budget(job.timeout, true);
3287            retry.stem = format!("{}-continue{attempts}", job.stem);
3288            let cache = self.state.config.cache_dir();
3289            let ctx = WaveCtx {
3290                carry_seats: false,
3291                pauses: Some((&self.pause, &self.interrupt)),
3292                run: run_id,
3293                node: "fix",
3294                prompts,
3295                cache: cache.as_deref(),
3296                round: Some(round),
3297            };
3298            let (resumed_seat, resumed_out) = run_one(
3299                retry,
3300                Arc::clone(&self.sem),
3301                &ctx,
3302                &mut self.state,
3303                attempts,
3304            )
3305            .await;
3306            seat = resumed_seat;
3307            match resumed_out {
3308                AgentOutcome::Ok(o) => {
3309                    cumulative_wait_ms += o.duration_ms;
3310                    match verdict::extract_json::<FixReport>(&o.text) {
3311                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3312                            self.state.event(
3313                                "fix",
3314                                format!(
3315                                    "round {round}: fixer's adoption report recovered after \
3316                                     {attempts} continuation(s)"
3317                                ),
3318                            );
3319                            return (
3320                                seat,
3321                                Some(report),
3322                                None,
3323                                ContinuationRecord {
3324                                    attempts,
3325                                    cumulative_wait_ms,
3326                                    outcome: ContinuationOutcome::Resumed,
3327                                },
3328                            );
3329                        }
3330                        // The report parsed, but this same reply's own
3331                        // CommandEvidence — the identical record `state.jobs`
3332                        // renders — names a command whose CLI never
3333                        // confirmed an exit status. Read together, that is
3334                        // not a resolved answer: keep nudging rather than
3335                        // accept a report standing next to a command the
3336                        // seat's own CLI cannot vouch for.
3337                        Ok(_) => {
3338                            last_err = "the reply parsed, but it reported a command whose own CLI \
3339                                 never confirmed an exit status"
3340                                .to_owned();
3341                        }
3342                        Err(e) => last_err = e.to_string(),
3343                    }
3344                }
3345                AgentOutcome::Quota(o) => {
3346                    cumulative_wait_ms += o.duration_ms;
3347                    self.state.quota.push(QuotaLoss {
3348                        seat: seat.key.clone(),
3349                        node: "fix".to_owned(),
3350                        at: Timestamp::now(),
3351                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3352                    });
3353                    self.state.event(
3354                        "fix",
3355                        format!(
3356                            "round {round}: continuation rate limited (quota); not retrying now"
3357                        ),
3358                    );
3359                    return (
3360                        seat,
3361                        None,
3362                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3363                        ContinuationRecord {
3364                            attempts,
3365                            cumulative_wait_ms,
3366                            outcome: ContinuationOutcome::QuotaLost,
3367                        },
3368                    );
3369                }
3370                AgentOutcome::Dropped(o) => {
3371                    cumulative_wait_ms += o.duration_ms;
3372                    let why = o
3373                        .dropped
3374                        .as_ref()
3375                        .map(|d| d.why.as_str())
3376                        .unwrap_or("the CLI ended the stream without delivering its answer");
3377                    last_err = format!("the CLI dropped the stream ({why})");
3378                }
3379                AgentOutcome::Failed(e) => last_err = e,
3380                // The continuation was never started; the report stays lost
3381                // and the caller records that, without any handover.
3382                AgentOutcome::ResourceBlocked(e) => {
3383                    return (
3384                        seat,
3385                        None,
3386                        Some(format!("the continuation was not started: {e}")),
3387                        ContinuationRecord {
3388                            attempts: attempts.saturating_sub(1),
3389                            cumulative_wait_ms,
3390                            outcome: ContinuationOutcome::NoSession,
3391                        },
3392                    );
3393                }
3394            }
3395        }
3396    }
3397
3398    fn after_implement(&mut self) -> Result<()> {
3399        if !self.state.candidate_preservation.is_empty() {
3400            bail!("candidate preservation incomplete; judging is unsafe");
3401        }
3402        // Scan every candidate patch once the set is complete.
3403        if self.state.leaks.is_empty() {
3404            let cfg = self.state.config.blind.clone();
3405            let mut leaks = Vec::new();
3406            for c in &self.state.candidates {
3407                let Some(patch) =
3408                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3409                else {
3410                    continue;
3411                };
3412                leaks.extend(blind::scan(
3413                    &format!("candidate {} patch", c.label),
3414                    &patch,
3415                    &cfg.vendor_tokens,
3416                ));
3417            }
3418            if !leaks.is_empty() {
3419                let summary = leaks
3420                    .iter()
3421                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3422                    .collect::<Vec<_>>()
3423                    .join(", ");
3424                match cfg.on_leak {
3425                    LeakPolicy::Fail => {
3426                        self.state.status = RunStatus::Failed;
3427                        self.state
3428                            .event("blind", format!("vendor text in a patch: {summary}"));
3429                        self.state.leaks = leaks;
3430                        self.state.save()?;
3431                        self.settle_questions();
3432                        bail!(
3433                            "blind.on_leak = \"fail\" and vendor text reached a \
3434                             judged patch: {summary}"
3435                        );
3436                    }
3437                    LeakPolicy::Redact => self.state.event(
3438                        "blind",
3439                        format!("redacting vendor text for judging: {summary}"),
3440                    ),
3441                    LeakPolicy::Warn => self.state.event(
3442                        "blind",
3443                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3444                    ),
3445                }
3446                self.state.leaks = leaks;
3447            }
3448        }
3449
3450        if self.state.viable().is_empty() {
3451            if self.state.all_candidates_verified_noop() {
3452                // Every candidate agreed, with evidence the adoption guard
3453                // accepted, that nothing belongs in this worktree. That is
3454                // not the same fact as a candidate that simply failed to
3455                // write anything, and settling it as an ordinary `Failed`
3456                // (see `SCHEMA`'s doc for schema 10) is what let two of
3457                // task 391f's attempts burn a retry each re-discovering the
3458                // same already-landed fix. Terminal either way, so `judge`
3459                // must never run over an empty candidate set — unlike the
3460                // `Failed` branch below this returns `Ok`, not an error:
3461                // nothing here failed.
3462                self.state.status = RunStatus::VerifiedNoop;
3463                self.state.save()?;
3464                self.settle_questions();
3465                return Ok(());
3466            }
3467            self.state.status = RunStatus::Failed;
3468            self.state.save()?;
3469            self.settle_questions();
3470            bail!("no candidate produced a change; nothing to judge");
3471        }
3472        self.state.status = RunStatus::Judging;
3473        self.state.save()?;
3474        Ok(())
3475    }
3476
3477    // --------------------------------------------------------------- judge
3478
3479    async fn judge(&mut self) -> Result<()> {
3480        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3481        // agent files with `magi task add` name the run that paid for it. The
3482        // prompt overlay is cloned alongside it because the waves borrow it
3483        // while `self` is mutably borrowed by the node's own bookkeeping.
3484        let run_id = self.state.id.clone();
3485        let prompts = self.state.config.prompts.clone();
3486        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3487            return Ok(());
3488        }
3489        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3490        if viable.len() == 1 {
3491            // Recorded so this is a one-time event: `judgements` stays empty
3492            // either way, which without this flag is indistinguishable from
3493            // "not yet judged" on the next reentry — and status is left
3494            // untouched, so a later node's conclusion (e.g. `Blocked` after
3495            // the review budget ran out) survives a resume instead of being
3496            // clobbered back to `Judging` by this node running again.
3497            self.state.judge_skipped = true;
3498            self.state.event(
3499                "judge",
3500                format!(
3501                    "only candidate {} produced a change; judging skipped",
3502                    viable[0].label
3503                ),
3504            );
3505            self.state.save()?;
3506            return Ok(());
3507        }
3508        self.state.status = RunStatus::Judging;
3509
3510        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3511        let language = self.state.config.graph.language.clone();
3512        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3513        let sessions = self.state.config.graph.sessions;
3514        let artifacts = agent::artifacts_dir(&self.state.dir());
3515        let root = self.state.worktree_root();
3516        let base_short = short(&self.state.base_commit);
3517
3518        let mut jobs = Vec::new();
3519        let mut orders = Vec::new();
3520        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3521            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3522            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3523            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3524            let seat_key = format!("judge-{}", j + 1);
3525            let seat = self.seat(&seat_key, &spec.id);
3526            jobs.push(SeatJob {
3527                prompt: prompt::judge(
3528                    &self.state.instruction,
3529                    &views,
3530                    self.roles.judges.len(),
3531                    &base_short,
3532                    &language,
3533                ),
3534                spec,
3535                seat,
3536                cwd: root.join(format!("judge-{}", j + 1)),
3537                timeout,
3538                allow_write: false,
3539                sessions,
3540                artifacts: artifacts.clone(),
3541                stem: format!("judge-{}", j + 1),
3542                handover: None,
3543            });
3544        }
3545
3546        self.state.event(
3547            "judge",
3548            format!(
3549                "{} judges ranking {} candidates blind",
3550                jobs.len(),
3551                viable.len()
3552            ),
3553        );
3554        let labels_for_check = labels.clone();
3555        let mut quota_losses = Vec::new();
3556        let cache = self.state.config.cache_dir();
3557        let ctx = WaveCtx {
3558            carry_seats: false,
3559            pauses: Some((&self.pause, &self.interrupt)),
3560            run: &run_id,
3561            node: "judge",
3562            prompts: &prompts,
3563            cache: cache.as_deref(),
3564            round: None,
3565        };
3566        let results = ask_json_wave::<Ranking>(
3567            jobs,
3568            Arc::clone(&self.sem),
3569            self.state.config.graph.retries,
3570            &self.roles.judge_roster,
3571            &ctx,
3572            &mut quota_losses,
3573            &mut self.state,
3574            &move |r: &Ranking| r.validate(&labels_for_check),
3575        )
3576        .await;
3577        if self.state.resource_block.is_some() {
3578            return self.stop_for_resource();
3579        }
3580        self.state.quota.extend(quota_losses);
3581
3582        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3583            let agent_id = seat.agent.clone();
3584            self.state.seats.insert(seat.key.clone(), seat);
3585            let mut record = Judgement {
3586                judge: j + 1,
3587                seat: format!("judge-{}", j + 1),
3588                agent: agent_id,
3589                ranking: Vec::new(),
3590                reasons: BTreeMap::new(),
3591                confidence: None,
3592                strengths: BTreeMap::new(),
3593                order: orders[j].clone(),
3594                failed: None,
3595                duration_ms: 0,
3596            };
3597            match res {
3598                Ok((ranking, out)) => {
3599                    record.ranking = ranking.normalized();
3600                    record.reasons = ranking.reasons;
3601                    record.strengths = ranking.strengths;
3602                    record.confidence = ranking.confidence;
3603                    record.duration_ms = out.duration_ms;
3604                    self.state.event(
3605                        "judge",
3606                        format!(
3607                            "judge {} ranked {}",
3608                            j + 1,
3609                            record.ranking.iter().collect::<String>()
3610                        ),
3611                    );
3612                }
3613                Err(e) => {
3614                    record.failed = Some(e.to_string());
3615                    self.state
3616                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3617                }
3618            }
3619            self.state.judgements.push(record);
3620            self.state.save()?;
3621        }
3622        Ok(())
3623    }
3624
3625    // ---------------------------------------------------------- deliberate
3626
3627    async fn deliberate(&mut self) -> Result<()> {
3628        if self.hold_base_sync().await? {
3629            return Ok(());
3630        }
3631        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3632        // agent files with `magi task add` name the run that paid for it. The
3633        // prompt overlay is cloned alongside it because the waves borrow it
3634        // while `self` is mutably borrowed by the node's own bookkeeping.
3635        let run_id = self.state.id.clone();
3636        let prompts = self.state.config.prompts.clone();
3637        if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3638            return Ok(());
3639        }
3640        let tops: Vec<char> = self
3641            .state
3642            .judgements
3643            .iter()
3644            .filter_map(|j| j.ranking.first().copied())
3645            .collect();
3646        let rounds = self.state.config.graph.deliberate_rounds;
3647        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3648            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3649                self.state.event(
3650                    "deliberate",
3651                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3652                );
3653            }
3654            self.state.status = RunStatus::Voting;
3655            self.state.save()?;
3656            return Ok(());
3657        }
3658
3659        self.state.status = RunStatus::Deliberating;
3660        self.state.event(
3661            "deliberate",
3662            format!(
3663                "split: first choices were {} — opening {rounds} round(s)",
3664                tops.iter().collect::<String>()
3665            ),
3666        );
3667
3668        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3669        let language = self.state.config.graph.language.clone();
3670        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3671        let sessions = self.state.config.graph.sessions;
3672        let artifacts = agent::artifacts_dir(&self.state.dir());
3673        let root = self.state.worktree_root();
3674        let base_short = short(&self.state.base_commit);
3675
3676        // Judges argue in sequence so that a turn can answer the one before it;
3677        // that is the difference between deliberation and three parallel
3678        // monologues.
3679        for round in 1..=rounds {
3680            let mut turns: Vec<DeliberationTurn> = Vec::new();
3681            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3682                if self.state.judgements[j].failed.is_some() {
3683                    continue;
3684                }
3685                let seat_key = format!("judge-{}", j + 1);
3686                let spec = self.occupant(&seat_key, spec);
3687                let mut seat = self.seat(&seat_key, &spec.id);
3688                let transcript = self.transcript(&turns, j);
3689                let build = |context: Option<&str>| {
3690                    prompt::deliberate(
3691                        &self.state.instruction,
3692                        context,
3693                        &transcript,
3694                        round,
3695                        rounds,
3696                        &language,
3697                    )
3698                };
3699                let block = self.candidate_block(&viable, &base_short);
3700                let full = build(Some(&block));
3701                let text = if has_context(&spec, &seat, sessions) {
3702                    build(None)
3703                } else {
3704                    full.clone()
3705                };
3706                let job = SeatJob {
3707                    spec,
3708                    seat: seat.clone(),
3709                    prompt: text,
3710                    cwd: root.join(format!("judge-{}", j + 1)),
3711                    timeout,
3712                    allow_write: false,
3713                    sessions,
3714                    artifacts: artifacts.clone(),
3715                    stem: format!("delib-{round}-judge-{}", j + 1),
3716                    handover: Some(full),
3717                };
3718                let cache = self.state.config.cache_dir();
3719                let ctx = WaveCtx {
3720                    carry_seats: false,
3721                    pauses: Some((&self.pause, &self.interrupt)),
3722                    run: &run_id,
3723                    node: "deliberate",
3724                    prompts: &prompts,
3725                    cache: cache.as_deref(),
3726                    round: None,
3727                };
3728                // A turn is never nudged (`retries` 0): a failed seat is
3729                // handed to the next roster agent, which gets the full
3730                // context. An empty answer is a turn, not a failure.
3731                let mut losses = Vec::new();
3732                let mut results = ask_wave_with::<String>(
3733                    vec![job],
3734                    Arc::clone(&self.sem),
3735                    0,
3736                    &self.roles.judge_roster,
3737                    &ctx,
3738                    &mut losses,
3739                    &mut self.state,
3740                    &|text: &str| {
3741                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3742                    },
3743                )
3744                .await;
3745                self.state.quota.extend(losses);
3746                let (updated, res, _) = results.pop().expect("one job in, one result out");
3747                seat = updated;
3748                let agent_id = seat.agent.clone();
3749                self.state.seats.insert(seat.key.clone(), seat);
3750                let body = match res {
3751                    Ok((body, _)) => body,
3752                    // Skip the seat; a CLI's raw error JSON is never read as
3753                    // this judge's position.
3754                    Err(e) => {
3755                        self.state
3756                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3757                        continue;
3758                    }
3759                };
3760                let tentative = verdict::extract_json::<Position>(&body)
3761                    .ok()
3762                    .and_then(|p| p.tentative)
3763                    .and_then(|s| s.trim().chars().next())
3764                    .map(|c| c.to_ascii_uppercase());
3765                self.state.event(
3766                    "deliberate",
3767                    format!(
3768                        "round {round}: judge {} now favours {}",
3769                        j + 1,
3770                        tentative.map_or("—".to_owned(), |c| c.to_string())
3771                    ),
3772                );
3773                turns.push(DeliberationTurn {
3774                    judge: j + 1,
3775                    agent: agent_id,
3776                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3777                    tentative,
3778                });
3779            }
3780            self.state
3781                .deliberation
3782                .push(DeliberationRound { round, turns });
3783            self.state.save()?;
3784        }
3785
3786        self.state.status = RunStatus::Voting;
3787        self.state.save()?;
3788        Ok(())
3789    }
3790
3791    // ---------------------------------------------------------------- vote
3792
3793    async fn vote(&mut self) -> Result<()> {
3794        if self.hold_base_sync().await? {
3795            return Ok(());
3796        }
3797        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3798        // agent files with `magi task add` name the run that paid for it. The
3799        // prompt overlay is cloned alongside it because the waves borrow it
3800        // while `self` is mutably borrowed by the node's own bookkeeping.
3801        let run_id = self.state.id.clone();
3802        let prompts = self.state.config.prompts.clone();
3803        if !self.state.votes.is_empty() || self.state.tally.is_some() {
3804            return Ok(());
3805        }
3806        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3807        if viable.len() == 1 {
3808            return Ok(());
3809        }
3810        self.state.status = RunStatus::Voting;
3811
3812        let language = self.state.config.graph.language.clone();
3813        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3814        let sessions = self.state.config.graph.sessions;
3815        let artifacts = agent::artifacts_dir(&self.state.dir());
3816        let root = self.state.worktree_root();
3817        let base_short = short(&self.state.base_commit);
3818        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3819
3820        let mut jobs = Vec::new();
3821        let mut seats_at = Vec::new();
3822        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3823            if self
3824                .state
3825                .judgements
3826                .get(j)
3827                .is_some_and(|r| r.failed.is_some())
3828            {
3829                continue;
3830            }
3831            let seat_key = format!("judge-{}", j + 1);
3832            let spec = self.occupant(&seat_key, spec);
3833            let seat = self.seat(&seat_key, &spec.id);
3834            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3835            let text = if has_context(&spec, &seat, sessions) {
3836                prompt::final_vote(&viable, &language)
3837            } else {
3838                full.clone()
3839            };
3840            jobs.push(SeatJob {
3841                spec,
3842                seat,
3843                prompt: text,
3844                cwd: root.join(format!("judge-{}", j + 1)),
3845                timeout,
3846                allow_write: false,
3847                sessions,
3848                artifacts: artifacts.clone(),
3849                stem: format!("vote-judge-{}", j + 1),
3850                handover: Some(full),
3851            });
3852            seats_at.push(j);
3853        }
3854
3855        self.state.event(
3856            "vote",
3857            format!(
3858                "collecting {} final votes one by one, privately",
3859                jobs.len()
3860            ),
3861        );
3862        let allowed = viable.clone();
3863        let mut quota_losses = Vec::new();
3864        let cache = self.state.config.cache_dir();
3865        let ctx = WaveCtx {
3866            carry_seats: false,
3867            pauses: Some((&self.pause, &self.interrupt)),
3868            run: &run_id,
3869            node: "vote",
3870            prompts: &prompts,
3871            cache: cache.as_deref(),
3872            round: None,
3873        };
3874        let results = ask_json_wave::<FinalVote>(
3875            jobs,
3876            Arc::clone(&self.sem),
3877            self.state.config.graph.retries,
3878            &self.roles.judge_roster,
3879            &ctx,
3880            &mut quota_losses,
3881            &mut self.state,
3882            &move |v: &FinalVote| match v.label() {
3883                Some(c) if allowed.contains(&c) => Ok(()),
3884                other => bail!("vote {other:?} is not one of {allowed:?}"),
3885            },
3886        )
3887        .await;
3888        if self.state.resource_block.is_some() {
3889            return self.stop_for_resource();
3890        }
3891        self.state.quota.extend(quota_losses);
3892
3893        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3894            let agent_id = seat.agent.clone();
3895            self.state.seats.insert(seat.key.clone(), seat);
3896            let initial = self
3897                .state
3898                .judgements
3899                .get(j)
3900                .and_then(|r| r.ranking.first().copied());
3901            let mut record = VoteRecord {
3902                judge: j + 1,
3903                agent: agent_id,
3904                vote: None,
3905                reason: String::new(),
3906                changed: false,
3907            };
3908            match res {
3909                Ok((v, _)) => {
3910                    record.vote = v.label();
3911                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3912                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3913                    self.state.event(
3914                        "vote",
3915                        format!(
3916                            "judge {} voted {}{}",
3917                            j + 1,
3918                            record.vote.unwrap_or('?'),
3919                            if record.changed { " (changed)" } else { "" }
3920                        ),
3921                    );
3922                }
3923                Err(e) => {
3924                    self.state
3925                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3926                }
3927            }
3928            self.state.votes.push(record);
3929            self.state.save()?;
3930        }
3931        Ok(())
3932    }
3933
3934    // --------------------------------------------------------------- tally
3935
3936    fn tally(&mut self) -> Result<()> {
3937        if self.state.tally.is_some() {
3938            return Ok(());
3939        }
3940        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3941        let tops: Vec<char> = self
3942            .state
3943            .judgements
3944            .iter()
3945            .filter_map(|j| j.ranking.first().copied())
3946            .collect();
3947        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3948
3949        // A judge whose private vote failed still counted once, in the initial
3950        // ranking; using it beats discarding a whole seat.
3951        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3952        let mut cast: Vec<char> = Vec::new();
3953        for (i, j) in self.state.judgements.iter().enumerate() {
3954            let vote = self
3955                .state
3956                .votes
3957                .iter()
3958                .find(|v| v.judge == i + 1)
3959                .and_then(|v| v.vote)
3960                .or_else(|| j.ranking.first().copied());
3961            if let Some(v) = vote {
3962                *first_choice.entry(v).or_insert(0) += 1;
3963                cast.push(v);
3964            }
3965        }
3966
3967        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3968        for j in &self.state.judgements {
3969            let n = j.ranking.len();
3970            for (pos, label) in j.ranking.iter().enumerate() {
3971                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3972            }
3973        }
3974
3975        let best = first_choice.values().copied().max().unwrap_or(0);
3976        let mut leaders: Vec<char> = first_choice
3977            .iter()
3978            .filter(|(_, v)| **v == best)
3979            .map(|(k, _)| *k)
3980            .collect();
3981        let mut tie_break = None;
3982        if leaders.len() > 1 {
3983            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3984            let borda_leaders: Vec<char> = leaders
3985                .iter()
3986                .copied()
3987                .filter(|l| borda[l] == top_borda)
3988                .collect();
3989            tie_break = Some(if borda_leaders.len() == 1 {
3990                format!(
3991                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3992                    leaders.len()
3993                )
3994            } else {
3995                format!(
3996                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3997                    leaders.len()
3998                )
3999            });
4000            leaders = borda_leaders;
4001            leaders.sort_unstable();
4002        }
4003        let winner = *leaders
4004            .first()
4005            .or(viable.first())
4006            .context("no candidate to declare a winner from")?;
4007
4008        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4009        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4010        let deliberated = !self.state.deliberation.is_empty();
4011
4012        // Whose verdict is this? A rate-limited seat is absent even if it
4013        // ranked before the limit hit, so presence is measured against the
4014        // recorded losses, not just "did a ranking ever appear".
4015        let quota_seats: std::collections::BTreeSet<&str> =
4016            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4017        let mut present = 0usize;
4018        for (i, j) in self.state.judgements.iter().enumerate() {
4019            if quota_seats.contains(j.seat.as_str()) {
4020                continue;
4021            }
4022            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4023            let voted = self
4024                .state
4025                .votes
4026                .iter()
4027                .any(|v| v.judge == i + 1 && v.vote.is_some());
4028            if ranked || voted {
4029                present += 1;
4030            }
4031        }
4032        // Strict majority of the configured panel. A bare majority is real
4033        // signal we can act on, while a minority verdict must never stand in
4034        // for a healthy one. A one-candidate run needs no panel at all, and
4035        // `judges` stays `0` rather than the roster size a panel that never
4036        // sat would otherwise be credited with.
4037        let needs_quorum = viable.len() > 1;
4038        let judges_total = if needs_quorum {
4039            self.roles.judges.len()
4040        } else {
4041            0
4042        };
4043        let quorum = if needs_quorum {
4044            judges_total / 2 + 1
4045        } else {
4046            0
4047        };
4048        let met_quorum = !needs_quorum || present >= quorum;
4049        let uncontested = (!needs_quorum).then(|| {
4050            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4051        });
4052
4053        self.state.event(
4054            "tally",
4055            match &uncontested {
4056                Some(reason) => format!("winner {winner} — {reason}"),
4057                None => format!(
4058                    "winner {winner} — votes {} | initial {} | {} changed | \
4059                     {present}/{judges_total} judges{}",
4060                    first_choice
4061                        .iter()
4062                        .map(|(k, v)| format!("{k}:{v}"))
4063                        .collect::<Vec<_>>()
4064                        .join(" "),
4065                    if unanimous_initial {
4066                        "unanimous"
4067                    } else {
4068                        "split"
4069                    },
4070                    changed_votes,
4071                    if met_quorum {
4072                        String::new()
4073                    } else {
4074                        format!(" — below quorum ({quorum} required)")
4075                    },
4076                ),
4077            },
4078        );
4079        if !met_quorum {
4080            self.state.event(
4081                "stall",
4082                format!(
4083                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4084                     the run stops here, resumable"
4085                ),
4086            );
4087        }
4088        self.state.tally = Some(Tally {
4089            first_choice,
4090            borda,
4091            winner,
4092            rankings: tops.len(),
4093            unanimous_initial,
4094            deliberated,
4095            changed_votes,
4096            unanimous_final,
4097            tie_break,
4098            judges: judges_total,
4099            present,
4100            quorum,
4101            met_quorum,
4102            uncontested,
4103        });
4104        self.state.status = if met_quorum {
4105            RunStatus::Reviewing
4106        } else {
4107            RunStatus::Stalled
4108        };
4109        self.state.save()?;
4110        Ok(())
4111    }
4112
4113    // ------------------------------------------------------------- recover
4114
4115    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4116    /// resumed toward completion once the transient cause clears.
4117    ///
4118    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4119    /// it toward the quorum, which is exactly the set of seats whose absence
4120    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4121    /// not depend on which node happened to hit the limit), or an ordinary
4122    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4123    /// seat is never disturbed.
4124    ///
4125    /// A seat that now answers with a usable ranking is "recovered": its
4126    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4127    /// `tally` counts it present again), and its vote re-collected. A seat that
4128    /// still fails keeps its loss and stays absent.
4129    ///
4130    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4131    /// to review/gate/merge), `false` when it is still below quorum (the run
4132    /// stays `Stalled`, still resumable for a later retry).
4133    #[allow(clippy::too_many_lines)]
4134    async fn recover_stall(&mut self) -> Result<bool> {
4135        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4136        // agent files with `magi task add` name the run that paid for it. The
4137        // prompt overlay is cloned alongside it because the waves borrow it
4138        // while `self` is mutably borrowed by the node's own bookkeeping.
4139        let run_id = self.state.id.clone();
4140        let prompts = self.state.config.prompts.clone();
4141        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4142        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4143        // is what keeps a non-quota collapse — or a quota loss recorded at the
4144        // deliberate node — from being a permanent dead-end on `--resume`.
4145        let quota_seats: BTreeSet<&str> =
4146            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4147        let absent: Vec<String> = self
4148            .state
4149            .judgements
4150            .iter()
4151            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4152            .map(|j| j.seat.clone())
4153            .collect();
4154        if absent.is_empty() {
4155            return Ok(false);
4156        }
4157        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4158        if viable.len() <= 1 {
4159            return Ok(false);
4160        }
4161        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4162        let language = self.state.config.graph.language.clone();
4163        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4164        let sessions = self.state.config.graph.sessions;
4165        let artifacts = agent::artifacts_dir(&self.state.dir());
4166        let root = self.state.worktree_root();
4167        let base_short = short(&self.state.base_commit);
4168        let candidates: Vec<Candidate> = viable.clone();
4169
4170        // Map each absent seat key to its 0-based position in `roles.judges`.
4171        let mut positions: Vec<usize> = absent
4172            .iter()
4173            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4174            .collect();
4175        if positions.is_empty() {
4176            return Ok(false);
4177        }
4178        positions.sort_unstable();
4179        positions.dedup();
4180
4181        // Re-rank the lost seats, one blind prompt each.
4182        let mut judge_jobs = Vec::new();
4183        for &j in &positions {
4184            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4185            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4186            let seat_key = format!("judge-{}", j + 1);
4187            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4188            let seat = self.seat(&seat_key, &spec.id);
4189            judge_jobs.push(SeatJob {
4190                spec,
4191                seat,
4192                prompt: prompt::judge(
4193                    &self.state.instruction,
4194                    &views,
4195                    self.roles.judges.len(),
4196                    &base_short,
4197                    &language,
4198                ),
4199                cwd: root.join(seat_key),
4200                timeout,
4201                allow_write: false,
4202                sessions,
4203                artifacts: artifacts.clone(),
4204                stem: format!("judge-{}-recover", j + 1),
4205                handover: None,
4206            });
4207        }
4208
4209        let labels_for_check = labels.clone();
4210        let mut judge_losses = Vec::new();
4211        let retries = self.state.config.graph.retries;
4212        let cache = self.state.config.cache_dir();
4213        let ctx = WaveCtx {
4214            carry_seats: false,
4215            pauses: Some((&self.pause, &self.interrupt)),
4216            run: &run_id,
4217            node: "judge",
4218            prompts: &prompts,
4219            cache: cache.as_deref(),
4220            round: None,
4221        };
4222        let results = ask_json_wave::<Ranking>(
4223            judge_jobs,
4224            Arc::clone(&self.sem),
4225            retries,
4226            &self.roles.judge_roster,
4227            &ctx,
4228            &mut judge_losses,
4229            &mut self.state,
4230            &move |r: &Ranking| r.validate(&labels_for_check),
4231        )
4232        .await;
4233        if self.state.resource_block.is_some() {
4234            self.stop_for_resource()?;
4235            return Ok(false);
4236        }
4237
4238        // Refresh the judgement of every seat that ranked again.
4239        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4240        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4241            let agent_id = seat.agent.clone();
4242            self.state.seats.insert(seat.key.clone(), seat);
4243            let record = &mut self.state.judgements[j];
4244            match res {
4245                Ok((ranking, out)) => {
4246                    record.agent = agent_id;
4247                    record.ranking = ranking.normalized();
4248                    record.reasons = ranking.reasons;
4249                    record.strengths = ranking.strengths;
4250                    record.confidence = ranking.confidence;
4251                    record.failed = None;
4252                    record.duration_ms = out.duration_ms;
4253                    recovered.insert(j);
4254                    self.state.event(
4255                        "recover",
4256                        format!("judge {} ranked again after the limit", j + 1),
4257                    );
4258                }
4259                Err(e) => {
4260                    self.state
4261                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4262                }
4263            }
4264        }
4265
4266        // Re-ask the votes of the seats that recovered a ranking.
4267        let mut vote_jobs = Vec::new();
4268        let mut vote_pos: Vec<usize> = Vec::new();
4269        for &j in &recovered {
4270            let seat_key = format!("judge-{}", j + 1);
4271            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4272            let seat = self.seat(&seat_key, &spec.id);
4273            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4274            let text = if has_context(&spec, &seat, sessions) {
4275                prompt::final_vote(&labels, &language)
4276            } else {
4277                full.clone()
4278            };
4279            vote_jobs.push(SeatJob {
4280                spec,
4281                seat,
4282                prompt: text,
4283                cwd: root.join(seat_key),
4284                timeout,
4285                allow_write: false,
4286                sessions,
4287                artifacts: artifacts.clone(),
4288                stem: format!("vote-judge-{}-recover", j + 1),
4289                handover: Some(full),
4290            });
4291            vote_pos.push(j);
4292        }
4293        let allowed = labels.clone();
4294        let mut vote_losses = Vec::new();
4295        let vote_retries = self.state.config.graph.retries;
4296        let vote_cache = self.state.config.cache_dir();
4297        let ctx = WaveCtx {
4298            carry_seats: false,
4299            pauses: Some((&self.pause, &self.interrupt)),
4300            run: &run_id,
4301            node: "vote",
4302            prompts: &prompts,
4303            cache: vote_cache.as_deref(),
4304            round: None,
4305        };
4306        let votes = ask_json_wave::<FinalVote>(
4307            vote_jobs,
4308            Arc::clone(&self.sem),
4309            vote_retries,
4310            &self.roles.judge_roster,
4311            &ctx,
4312            &mut vote_losses,
4313            &mut self.state,
4314            &move |v: &FinalVote| match v.label() {
4315                Some(c) if allowed.contains(&c) => Ok(()),
4316                other => bail!("vote {other:?} is not one of {allowed:?}"),
4317            },
4318        )
4319        .await;
4320        if self.state.resource_block.is_some() {
4321            self.stop_for_resource()?;
4322            return Ok(false);
4323        }
4324        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4325            let agent_id = seat.agent.clone();
4326            self.state.seats.insert(seat.key.clone(), seat);
4327            match res {
4328                Ok((v, _)) => {
4329                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4330                        rec.vote = v.label();
4331                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4332                    } else {
4333                        self.state.votes.push(VoteRecord {
4334                            judge: j + 1,
4335                            agent: agent_id,
4336                            vote: v.label(),
4337                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4338                            changed: false,
4339                        });
4340                    }
4341                    self.state.event(
4342                        "recover",
4343                        format!("judge {} voted again after the limit", j + 1),
4344                    );
4345                }
4346                Err(e) => {
4347                    self.state
4348                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4349                }
4350            }
4351        }
4352
4353        // A seat that ranked again is present even if its re-vote failed —
4354        // `tally` falls back to the initial ranking's first choice — so clear
4355        // its quota loss. Seats that still fail keep theirs and stay absent.
4356        let recovered_keys: BTreeSet<String> = recovered
4357            .iter()
4358            .map(|&j| format!("judge-{}", j + 1))
4359            .collect();
4360        self.state
4361            .quota
4362            .retain(|q| !recovered_keys.contains(&q.seat));
4363        // A seat that hit the limit again is a fresh loss, not the old one:
4364        // replace the stale entry so the history stays one-per-seat and the
4365        // daemon can tell this attempt's loss from a previous session's.
4366        for loss in judge_losses.into_iter().chain(vote_losses) {
4367            if recovered_keys.contains(&loss.seat) {
4368                continue;
4369            }
4370            self.state.quota.retain(|q| q.seat != loss.seat);
4371            self.state.quota.push(loss);
4372        }
4373
4374        // Recompute the verdict from the refreshed panel.
4375        self.state.tally = None;
4376        self.tally()?;
4377        Ok(self
4378            .state
4379            .tally
4380            .as_ref()
4381            .map(|t| t.met_quorum)
4382            .unwrap_or(false))
4383    }
4384
4385    // ---------------------------------------------------------------- graft
4386
4387    /// Port the strengths the judges attributed to the losing candidates onto
4388    /// the winner, once, between the tally and the review loop.
4389    ///
4390    /// Best-effort by construction: it returns nothing and fails nothing. A
4391    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4392    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4393    /// the run carries on as if the phase were off. See [`graft`] for the
4394    /// decisions and `AGENTS.md` for the invariants.
4395    ///
4396    /// The one exception is a rollback that could not be confirmed: the
4397    /// rejected commit may still be the winner's tip, so the run is blocked
4398    /// (resumable; the record stays `Running`, which makes a resume retry the
4399    /// restore) instead of reviewing, gating and merging it.
4400    async fn graft(&mut self) -> Result<()> {
4401        let Err(e) = self.graft_inner().await else {
4402            return Ok(());
4403        };
4404        if e.is::<crate::repo_lock::ResourceBlocked>() {
4405            return Err(e);
4406        }
4407        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4408            self.state.status = RunStatus::Blocked;
4409            self.state.event("graft", format!("blocked: {e:#}"));
4410            self.state.save()?;
4411            return Err(e);
4412        }
4413        // An error after the `Running` record was saved may have left the
4414        // fixer's work in the tree: the record must not stay unsettled
4415        // (a later resume would reset past the review fixes), so confirm the
4416        // restore now, or block.
4417        if let Some(rec) = self.state.graft.clone()
4418            && !rec.outcome.settled()
4419        {
4420            if let Some(winner) = self.state.winner().cloned() {
4421                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4422                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4423                        return Err(re);
4424                    }
4425                    self.state.status = RunStatus::Blocked;
4426                    self.state.event("graft", format!("blocked: {re:#}"));
4427                    self.state.save()?;
4428                    return Err(re);
4429                }
4430            }
4431            if let Some(g) = self.state.graft.as_mut() {
4432                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4433            }
4434        }
4435        self.state.event("graft", format!("skipped: {e:#}"));
4436        if let Err(e) = self.state.save() {
4437            tracing::warn!("save after graft: {e:#}");
4438        }
4439        Ok(())
4440    }
4441
4442    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4443    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4444    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4445        let _coordination = self.coordinate(wt).await?;
4446        crate::recovery::restore_repair(wt, pre)
4447            .await
4448            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4449    }
4450
4451    /// A skip is a run event, recorded once however often the node is re-entered.
4452    fn graft_skipped(&mut self, why: &str) {
4453        let message = format!("skipped: {why}");
4454        if !self
4455            .state
4456            .events
4457            .iter()
4458            .any(|e| e.node == "graft" && e.message == message)
4459        {
4460            self.state.event("graft", message);
4461        }
4462    }
4463
4464    async fn graft_inner(&mut self) -> Result<()> {
4465        // A round that was started and never finished (crash, kill): the
4466        // fixer may have left anything in the tree, so put the winner back.
4467        if let Some(rec) = self.state.graft.clone()
4468            && !rec.outcome.settled()
4469        {
4470            // Reviews exist only if the graft ended without its record being
4471            // settled (a swallowed error): the tree has moved on since, and
4472            // a reset to `pre` would discard the review fixes.
4473            if self.state.reviews.is_empty()
4474                && let Some(winner) = self.state.winner().cloned()
4475            {
4476                self.graft_restore(&winner.worktree, &rec.pre).await?;
4477            }
4478            if rec.outcome == GraftOutcome::WaitingForRestore {
4479                self.state.graft = None;
4480                self.state.save()?;
4481            } else {
4482                if let Some(g) = self.state.graft.as_mut() {
4483                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4484                }
4485                self.state.event(
4486                    "graft",
4487                    "rolled back: the previous round was interrupted; the winner is untouched",
4488                );
4489                self.state.save()?;
4490                return Ok(());
4491            }
4492        }
4493        let plan = match graft::plan(&self.state) {
4494            Ok(plan) => plan,
4495            Err(skip) => {
4496                if skip != graft::Skip::AlreadyDone {
4497                    self.graft_skipped(skip.reason());
4498                    self.state.save()?;
4499                }
4500                return Ok(());
4501            }
4502        };
4503        let Some(winner) = self.state.winner().cloned() else {
4504            return Ok(());
4505        };
4506        let wt = winner.worktree.clone();
4507        if !git::is_clean(&wt).await? {
4508            self.graft_skipped("the winner's worktree has uncommitted changes");
4509            self.state.save()?;
4510            return Ok(());
4511        }
4512        let pre = git::rev_parse(&wt, "HEAD").await?;
4513        // Saved before the fixer is called: a stop from here on is repaired
4514        // by the branch above on the next entry.
4515        self.state.graft = Some(GraftRecord {
4516            pre: pre.clone(),
4517            post: None,
4518            lines: None,
4519            limit: None,
4520            agent: None,
4521            outcome: GraftOutcome::Running,
4522            requested: graft::requested(&plan),
4523            adoption: None,
4524            adoption_error: None,
4525        });
4526        self.state.event(
4527            "graft",
4528            format!(
4529                "porting strengths of candidate(s) {} onto {}",
4530                plan.strengths
4531                    .iter()
4532                    .map(|(l, _, _)| l.to_string())
4533                    .collect::<Vec<_>>()
4534                    .join(", "),
4535                plan.winner
4536            ),
4537        );
4538        self.state.save()?;
4539
4540        let fix_prompt = prompt::graft(
4541            &self.state.instruction,
4542            plan.winner,
4543            &plan.strengths,
4544            &self.state.config.graph.language,
4545        );
4546        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4547        let sessions = self.state.config.graph.sessions;
4548        let artifacts = agent::artifacts_dir(&self.state.dir());
4549        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4550        let (_, seat, out) = self
4551            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4552                SeatJob {
4553                    prompt: fix_prompt.clone(),
4554                    spec,
4555                    seat,
4556                    cwd: wt.clone(),
4557                    timeout,
4558                    allow_write: true,
4559                    sessions,
4560                    artifacts: artifacts.clone(),
4561                    stem: "graft".to_owned(),
4562                    handover: None,
4563                }
4564            })
4565            .await;
4566        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4567            // Preserve the need to restore earlier fallback edits without
4568            // confusing an unstarted seat with an interrupted invocation.
4569            self.state.graft.as_mut().expect("graft record").outcome =
4570                GraftOutcome::WaitingForRestore;
4571            return self.stop_for_resource();
4572        }
4573        let mut why: Option<String> = None;
4574        match &out {
4575            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4576                why = Some("the fixer produced nothing usable".to_owned());
4577            }
4578            AgentOutcome::Ok(_) => {}
4579            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4580            AgentOutcome::Quota(o) => {
4581                self.state.quota.push(QuotaLoss {
4582                    seat: seat.key.clone(),
4583                    node: "graft".to_owned(),
4584                    at: Timestamp::now(),
4585                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4586                });
4587                why = Some("rate limited (quota)".to_owned());
4588            }
4589            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4590                why = Some(format!(
4591                    "the fixer failed: {}",
4592                    e.lines()
4593                        .next()
4594                        .unwrap_or("")
4595                        .chars()
4596                        .take(160)
4597                        .collect::<String>()
4598                ));
4599            }
4600        }
4601        let agent_id = seat.agent.clone();
4602        self.state.seats.insert(seat.key.clone(), seat);
4603
4604        let mut post = None;
4605        let mut lines = None;
4606        let _coordination = self.coordinate(&wt).await?;
4607        if why.is_none() {
4608            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4609            if let Ok(r) = &rescued {
4610                self.state.note_withheld("graft", &r.withheld);
4611            }
4612            let head = git::rev_parse(&wt, "HEAD").await?;
4613            // What is measured must be what gets reviewed and gated: a failed
4614            // rescue, or anything still uncommitted (withheld lockfiles
4615            // aside, which are never part of the tree), rejects the graft.
4616            let dirty = match &rescued {
4617                Ok(r) => git::status_porcelain(&wt)
4618                    .await
4619                    .map(|st| {
4620                        st.lines().any(|l| {
4621                            !r.withheld
4622                                .iter()
4623                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4624                        })
4625                    })
4626                    .unwrap_or(true),
4627                Err(_) => true,
4628            };
4629            if dirty {
4630                why = Some("the fixer left changes that could not be committed".to_owned());
4631            } else if head == pre {
4632                why = Some("the fixer changed nothing".to_owned());
4633            } else {
4634                lines = git::changed_lines(&wt, &pre, &head).await;
4635                post = Some(head);
4636            }
4637        }
4638        let (adoption, adoption_error) = match &out {
4639            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4640            _ => (None, None),
4641        };
4642        let requested = graft::requested(&plan);
4643        let outcome = match why {
4644            None => {
4645                self.state.event(
4646                    "graft",
4647                    format!(
4648                        "applied: {}",
4649                        lines.map_or_else(
4650                            || "changed lines unavailable".to_owned(),
4651                            |n| format!("{n} changed lines")
4652                        )
4653                    ),
4654                );
4655                GraftOutcome::Applied
4656            }
4657            Some(why) => {
4658                // Not settled until the restore is confirmed: the record
4659                // stays `Running`, so a resume retries it.
4660                crate::recovery::restore_repair(&wt, &pre)
4661                    .await
4662                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4663                self.state.event(
4664                    "graft",
4665                    format!("rolled back: {why}; the winner is untouched"),
4666                );
4667                GraftOutcome::RolledBack(why)
4668            }
4669        };
4670        self.state.graft = Some(GraftRecord {
4671            pre,
4672            post,
4673            lines,
4674            limit: None,
4675            agent: Some(agent_id),
4676            outcome,
4677            requested,
4678            adoption,
4679            adoption_error,
4680        });
4681        self.state.save()?;
4682        Ok(())
4683    }
4684
4685    // ----------------------------------------------------------------- fold
4686
4687    async fn fold_losers(&mut self) -> Result<()> {
4688        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4689            return Ok(());
4690        };
4691        if !self.state.candidate_preservation.is_empty() {
4692            bail!("candidate preservation incomplete; refusing cleanup");
4693        }
4694        let repo = self.state.repo.clone();
4695        let _coordination = self.coordinate(&repo).await?;
4696        let mut folded = Vec::new();
4697        for i in 0..self.state.candidates.len() {
4698            let c = &self.state.candidates[i];
4699            if c.label == winner || c.folded {
4700                continue;
4701            }
4702            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4703            remove_preserved_worktree(&repo, &wt).await?;
4704            if git::branch_exists(&repo, &branch).await?
4705                && !git::branch_delete(&repo, &branch).await?
4706            {
4707                bail!("candidate branch removal refused");
4708            }
4709            self.state.candidates[i].folded = true;
4710            self.state.save()?;
4711            folded.push(label.to_string());
4712        }
4713        // The judges are finished; their checkouts are pure cost from here.
4714        let root = self.state.worktree_root();
4715        for j in 1..=self.roles.judges.len() {
4716            let wt = root.join(format!("judge-{j}"));
4717            if wt.exists() {
4718                remove_preserved_worktree(&repo, &wt).await?;
4719            }
4720        }
4721        // The design-deliberation stage is finished by the time a tally
4722        // exists — same reasoning as the judges above.
4723        if self.state.config.graph.advise {
4724            for k in 1..=self.state.config.graph.advisors {
4725                let wt = root.join(format!("advisor-{k}"));
4726                if wt.exists() {
4727                    remove_preserved_worktree(&repo, &wt).await?;
4728                }
4729            }
4730        }
4731        if !folded.is_empty() {
4732            self.state
4733                .event("fold", format!("folded candidates {}", folded.join(", ")));
4734            self.state.save()?;
4735        }
4736        Ok(())
4737    }
4738
4739    // ------------------------------------------------------------ base sync
4740
4741    async fn hold_base_sync(&mut self) -> Result<bool> {
4742        self.hold_base_sync_as(None).await
4743    }
4744
4745    async fn hold_base_sync_kind(
4746        &mut self,
4747        kind: crate::base_sync_recovery::StopKind,
4748    ) -> Result<()> {
4749        self.hold_base_sync_as(Some(kind)).await?;
4750        Ok(())
4751    }
4752
4753    async fn hold_base_sync_as(
4754        &mut self,
4755        kind: Option<crate::base_sync_recovery::StopKind>,
4756    ) -> Result<bool> {
4757        if !self.state.base_sync_blocked() {
4758            self.state.base_sync_recovery.stop = None;
4759            return Ok(false);
4760        }
4761        let head = match self.state.winner() {
4762            Some(w) => git::rev_parse(&w.worktree, "HEAD")
4763                .await
4764                .unwrap_or_default(),
4765            None => String::new(),
4766        };
4767        self.state.record_base_sync_stop(&head);
4768        if let Some(kind) = kind {
4769            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4770        }
4771        self.state.save()?;
4772        Ok(true)
4773    }
4774
4775    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4776    /// anything verifies it.
4777    ///
4778    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4779    /// read whatever is checked out in the winner's worktree. Left alone that
4780    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4781    /// it when the run *branched* - and a run takes long enough that the base
4782    /// has usually moved by the time it gets here. A gate that ran there
4783    /// answers "green on the commit this run started from", not "green on
4784    /// what is about to land", and the difference showed up three times in
4785    /// one day as a green run whose merge would have reverted a file another
4786    /// pull request had already landed.
4787    ///
4788    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4789    /// `land::Step::Rebase` calls, rather than a second implementation of the
4790    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4791    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4792    /// doc warns about ("two answers to a question nobody notices until a
4793    /// diff is wrong").
4794    ///
4795    /// A conflict is not the end of the road: the standing rebase is handed
4796    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4797    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4798    /// land). Once it finishes, review and the gate run as usual on the
4799    /// rebased tree, which is where a breakage the new base caused is caught
4800    /// by the ordinary gate-fix round. magi resolves nothing itself.
4801    ///
4802    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4803    /// `state.base_sync.attempts`, is how many times the base is *rebased
4804    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4805    /// *conflict* was given to a fixer. When the fixer cannot finish the
4806    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4807    /// what was tried (rounds spent, paths still conflicted) and the branch
4808    /// and worktree stay exactly as they were - untouched, for a person to
4809    /// look at - which is also what makes re-entering this function
4810    /// afterwards a no-op instead of a second attempt at the same wall. A
4811    /// push failure ends the same way.
4812    async fn sync_to_base(&mut self) -> Result<()> {
4813        if self.state.status == RunStatus::AlreadyInBase {
4814            return Ok(());
4815        }
4816        self.hold_base_sync().await?;
4817        let conflicted = self
4818            .state
4819            .base_sync
4820            .as_ref()
4821            .is_some_and(|s| s.conflict.is_some());
4822        let Some(winner) = self.state.winner().cloned() else {
4823            return Ok(());
4824        };
4825
4826        let repo = self.state.repo.clone();
4827        let remote = self.state.config.merge.remote.clone();
4828        let base_branch = self.state.base_branch.clone();
4829        let tracking = format!("{remote}/{base_branch}");
4830
4831        let fetched_base =
4832            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4833        // No network, or the remote never had this branch: the run already
4834        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4835        // otherwise), and one that got this far is not blocked by a fetch
4836        // that fails now. It just has no newer tip to compare against.
4837        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4838            return Ok(());
4839        };
4840
4841        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4842        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4843        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4844
4845        // Before any rebase, and before a recorded conflict is honoured: a
4846        // branch whose change reached the base under other commit ids has
4847        // nothing to rebase and nothing to conflict with, and a run that
4848        // already stopped on that phantom conflict recovers here on resume.
4849        // `behind == 0` with head == tip is a branch the base has since taken
4850        // in whole, whether or not a conflict was ever recorded: the ancestry
4851        // proof must still run (`classify` ignores a head still on the start
4852        // commit).
4853        if (behind > 0 || conflicted || head == tip)
4854            && self
4855                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4856                .await?
4857        {
4858            return Ok(());
4859        }
4860        if conflicted && self.state.base_recovery_pending() {
4861            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4862            let owner =
4863                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4864            if !fetched_base
4865                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4866                || !git::is_clean(&winner.worktree).await?
4867                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4868            {
4869                return Ok(());
4870            }
4871        }
4872        let recovery = conflicted && self.state.consume_base_recovery(&head);
4873        if conflicted {
4874            // Consumption (including a refused changed HEAD) is durable before Git.
4875            self.state.save()?;
4876            if !recovery {
4877                return Ok(());
4878            }
4879        }
4880
4881        if behind == 0 {
4882            // A fixer-finished rebase moves the branch ref before the
4883            // winner's worktree is told (`sync_to_head` below). A run that
4884            // died in between resumes here with `behind == 0` and a tree still
4885            // holding the pre-rebase files, which review and the gate would
4886            // then read. That state is exactly: HEAD moved off the tip the
4887            // rebase started from, yet the tree is still identical to that
4888            // tip. A tree with edits of its own differs from it, so nothing
4889            // is thrown away.
4890            if let Some(from) = self
4891                .state
4892                .rebase_fixes
4893                .iter()
4894                .rev()
4895                .find_map(|r| r.from.clone())
4896                && from != head
4897                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4898                    .await
4899                    .is_ok_and(|o| o.ok())
4900            {
4901                git::sync_to_head(&winner.worktree).await?;
4902            }
4903            // An earlier attempt may have rebased the branch locally and died
4904            // before pushing it (only the fresh-rebase arm below pushes).
4905            // Publish it now, so the plain push at PR time is not refused as
4906            // a non-fast-forward. A run already holding a recorded conflict
4907            // never reaches here; that case is out of scope.
4908            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4909            if let Some(why) = &conflict {
4910                self.state.status = RunStatus::Blocked;
4911                self.state.event("land", why.clone());
4912            }
4913            self.state.base_sync = Some(BaseSync {
4914                tip,
4915                behind: 0,
4916                attempts: attempts + usize::from(recovery),
4917                conflict,
4918                already_in: None,
4919            });
4920            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4921                .await?;
4922            self.state.save()?;
4923            return Ok(());
4924        }
4925
4926        if attempts >= BASE_SYNC_ROUNDS && !recovery {
4927            let why = format!(
4928                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4929                 rebase(s); rebasing again would only race it",
4930                winner.branch
4931            );
4932            self.state.status = RunStatus::Blocked;
4933            self.state.base_sync = Some(BaseSync {
4934                tip,
4935                behind,
4936                attempts,
4937                conflict: Some(why.clone()),
4938                already_in: None,
4939            });
4940            self.state.event("land", why);
4941            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4942                .await?;
4943            self.state.save()?;
4944            return Ok(());
4945        }
4946
4947        self.state.event(
4948            "land",
4949            format!(
4950                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4951                winner.branch
4952            ),
4953        );
4954        self.state.save()?;
4955
4956        // The remote's copy of the branch, read now and only if the fetch
4957        // really succeeded (a stale tracking ref must never pin a lease). It is
4958        // pushed over after a rebase only when it is a commit this branch
4959        // already contains, by ancestry or by patch (an earlier rebase of ours
4960        // that never reached the remote): anything else is somebody else's work.
4961        let branch_tracking = format!("{remote}/{}", winner.branch);
4962        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4963        let remote_tip = if self.state.task_pr_branch().is_none()
4964            && matches!(&fetched_branch, Ok(o) if o.ok())
4965        {
4966            git::rev_parse(&repo, &branch_tracking).await.ok()
4967        } else {
4968            None
4969        };
4970        // A remote tip this branch does not contain is somebody else's work:
4971        // rebasing would leave a local tip that can never be pushed. Stop
4972        // before touching anything and say so.
4973        if let Some(theirs) = &remote_tip
4974            && !git::is_ancestor(&repo, theirs, &head).await
4975            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4976                .await
4977                .is_ok_and(|missing| missing.is_empty())
4978        {
4979            let why = format!(
4980                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4981                 them",
4982                short(theirs),
4983                winner.branch
4984            );
4985            self.state.status = RunStatus::Blocked;
4986            self.state.base_sync = Some(BaseSync {
4987                tip,
4988                behind,
4989                attempts: attempts + usize::from(recovery),
4990                conflict: Some(why.clone()),
4991                already_in: None,
4992            });
4993            self.state.event("land", why);
4994            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4995                .await?;
4996            self.state.save()?;
4997            return Ok(());
4998        }
4999
5000        let scratch = self.state.dir().join("base-sync");
5001        let rebased =
5002            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5003                .await
5004            {
5005                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5006                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5007                Err(e) => Err(e),
5008            };
5009        let attempts = attempts + 1;
5010        let stop_kind = match &rebased {
5011            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5012            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5013            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5014        };
5015        match rebased {
5016            Ok(None) => {
5017                // The branch ref moved, but a worktree that already had it
5018                // checked out (the winner's) was not told; sync its index and
5019                // files before anything reads them.
5020                git::sync_to_head(&winner.worktree).await?;
5021                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5022                let mut conflict = None;
5023                if let Some(pinned) = &remote_tip {
5024                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5025                    match pushed {
5026                        Ok(o) if o.ok() => self.state.event(
5027                            "land",
5028                            format!("pushed rebased {} to {remote}", winner.branch),
5029                        ),
5030                        Ok(o) => {
5031                            conflict = Some(format!(
5032                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5033                                winner.branch,
5034                                short(pinned),
5035                                o.stderr.chars().take(600).collect::<String>()
5036                            ));
5037                        }
5038                        Err(e) => {
5039                            conflict = Some(format!(
5040                                "rebased {} locally but could not push it: {e:#}",
5041                                winner.branch
5042                            ));
5043                        }
5044                    }
5045                }
5046                if let Some(why) = &conflict {
5047                    self.state.status = RunStatus::Blocked;
5048                    self.state.event("land", why.clone());
5049                }
5050                self.state.base_sync = Some(BaseSync {
5051                    tip: tip.clone(),
5052                    behind: 0,
5053                    attempts,
5054                    conflict,
5055                    already_in: None,
5056                });
5057                self.state
5058                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5059            }
5060            Ok(Some(conflict)) => {
5061                let why = format!(
5062                    "{} conflicts with {tracking} and did not rebase: {}",
5063                    winner.branch,
5064                    conflict.chars().take(600).collect::<String>()
5065                );
5066                self.state.status = RunStatus::Blocked;
5067                self.state.base_sync = Some(BaseSync {
5068                    tip,
5069                    behind,
5070                    attempts,
5071                    conflict: Some(why.clone()),
5072                    already_in: None,
5073                });
5074                self.state.event("land", why);
5075            }
5076            // A refused Git write or a busy repository: nothing is wrong with
5077            // the branch. The scratch rebase stays standing and the run
5078            // resumes into it (`base_sync` must not claim a conflict).
5079            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5080                self.state.save()?;
5081                return Ok(());
5082            }
5083            Err(e) => {
5084                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5085                self.state.status = RunStatus::Blocked;
5086                self.state.base_sync = Some(BaseSync {
5087                    tip,
5088                    behind,
5089                    attempts,
5090                    conflict: Some(why.clone()),
5091                    already_in: None,
5092                });
5093                self.state.event("land", why);
5094            }
5095        }
5096        if !self.state.base_sync_blocked() {
5097            self.state.base_sync_recovery.stop = None;
5098        }
5099        self.hold_base_sync_kind(stop_kind).await?;
5100        self.state.save()?;
5101        Ok(())
5102    }
5103
5104    /// Push a branch an earlier attempt rebased locally but never published,
5105    /// pinned to the remote tip read right after a successful fetch. Returns
5106    /// the reason when the run must stop; `None` when there was nothing to do
5107    /// (no remote copy, the same tip, or a remote copy this branch already
5108    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5109    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5110        // Task competitions publish only after the PR text gate, to their
5111        // public head; never resurrect a candidate branch on the remote.
5112        if self.state.task_pr_branch().is_some() {
5113            return None;
5114        }
5115        let repo = self.state.repo.clone();
5116        let remote = self.state.config.merge.remote.clone();
5117        let fetched = git::fetch(&repo, &remote, branch).await;
5118        if !matches!(&fetched, Ok(o) if o.ok()) {
5119            return None;
5120        }
5121        let branch_tracking = format!("{remote}/{branch}");
5122        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5123        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5124            return None;
5125        }
5126        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5127            .await
5128            .is_ok_and(|missing| missing.is_empty())
5129        {
5130            return Some(format!(
5131                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5132                 them",
5133                short(&theirs)
5134            ));
5135        }
5136        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5137            Ok(o) if o.ok() => {
5138                self.state
5139                    .event("land", format!("pushed rebased {branch} to {remote}"));
5140                None
5141            }
5142            Ok(o) => Some(format!(
5143                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5144                 someone may have pushed): {}",
5145                short(&theirs),
5146                o.stderr.chars().take(600).collect::<String>()
5147            )),
5148            Err(e) => Some(format!(
5149                "{branch} is rebased locally but could not be pushed: {e:#}"
5150            )),
5151        }
5152    }
5153
5154    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5155    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5156    /// whether it did.
5157    ///
5158    /// Checked only when the base is ahead of the branch. A failing check is
5159    /// "not proven" - the ordinary rebase path then decides - never a reason to
5160    /// stop the run.
5161    ///
5162    /// The remote copy of the branch is held to the same standard as the local
5163    /// one: if it carries a tip this worktree does not, that tip must itself be
5164    /// proven in the base, or nothing is settled (a pull request would
5165    /// otherwise be closed over commits nobody checked). The pull request is
5166    /// closed *before* the terminal status is saved; if that fails for a
5167    /// reason other than a refusal (no network, a `gh` error) the run is left
5168    /// `Blocked` with the reason as its conflict, which a resume retries -
5169    /// the same recovery a phantom conflict gets.
5170    async fn settle_already_in(
5171        &mut self,
5172        branch: &str,
5173        tip: &str,
5174        head: &str,
5175        attempts: usize,
5176        behind: usize,
5177    ) -> Result<bool> {
5178        let repo = self.state.repo.clone();
5179        let remote = self.state.config.merge.remote.clone();
5180        let start = self.state.base_commit.clone();
5181        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5182            Ok(Some(e)) => e,
5183            Ok(None) => return Ok(false),
5184            Err(e) => {
5185                tracing::warn!("already-in-base check for {branch}: {e:#}");
5186                return Ok(false);
5187            }
5188        };
5189        let mut verified = vec![head.to_owned()];
5190        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5191        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5192        if matches!(&fetched, Ok(o) if o.ok())
5193            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5194            && theirs != head
5195        {
5196            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5197                Ok(Some(_)) => verified.push(theirs),
5198                _ => return Ok(false),
5199            }
5200        }
5201        let base_branch = self.state.base_branch.clone();
5202        let message = format!(
5203            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5204             land",
5205            evidence.names(),
5206            evidence.proof.as_str()
5207        );
5208        let closed =
5209            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5210                .await;
5211        match closed {
5212            Ok(Ok(url)) => self
5213                .state
5214                .event("land", format!("closed {url}: superseded on {base_branch}")),
5215            Ok(Err(why)) => self
5216                .state
5217                .event("land", format!("did not close a pull request: {why}")),
5218            Err(e) => {
5219                let why = format!(
5220                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5221                     not be closed ({e:#}); resume to retry"
5222                );
5223                self.state.status = RunStatus::Blocked;
5224                self.state.base_sync = Some(BaseSync {
5225                    tip: tip.to_owned(),
5226                    behind,
5227                    attempts,
5228                    conflict: Some(why.clone()),
5229                    already_in: None,
5230                });
5231                self.state.event("land", why);
5232                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5233                    .await?;
5234                self.state.save()?;
5235                return Ok(true);
5236            }
5237        }
5238        self.state.status = RunStatus::AlreadyInBase;
5239        self.state.base_sync_recovery.stop = None;
5240        self.state.base_sync = Some(BaseSync {
5241            tip: tip.to_owned(),
5242            behind,
5243            attempts,
5244            conflict: None,
5245            already_in: Some(evidence),
5246        });
5247        self.state.event("land", message);
5248        self.state.save()?;
5249        self.settle_questions();
5250        Ok(true)
5251    }
5252
5253    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5254    /// last landed the winner on, once it has run, else the commit the run
5255    /// branched from.
5256    ///
5257    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5258    /// and `vote` all happen before there is a winner to rebase, so they
5259    /// compare every candidate against the branch point on purpose, and a
5260    /// base that moves after they are already done cannot change an answer
5261    /// they already gave.
5262    fn landing_base(&self) -> String {
5263        self.state
5264            .base_sync
5265            .as_ref()
5266            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5267    }
5268
5269    // ------------------------------------------------------- operator fix
5270
5271    /// Route specific, already-recorded review findings to a fixer for a
5272    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5273    /// entry point.
5274    ///
5275    /// Distinct from `review_loop`'s own fix step in three ways: it never
5276    /// runs a reviewer wave, it never spends review-round budget, and what
5277    /// happened is recorded as an [`OperatorFixRequest`] appended to
5278    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5279    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5280    /// and vote must never be rewritten to look like a manufactured blocking
5281    /// verdict.
5282    ///
5283    /// Only meaningful once review has actually concluded: `Ready` (handed
5284    /// off with findings still open, or simply concluded clean while minor
5285    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5286    /// gate failed). Everything else is refused: a run still in progress
5287    /// should simply be resumed, and a `Merged` run's branch has already
5288    /// landed — reopening *this* run's own record cannot change that, so the
5289    /// answer there is a fresh `magi review <branch>`.
5290    ///
5291    /// A real commit here re-verifies through a fresh, ordinary review-only
5292    /// run on the same branch ([`Self::review`]) rather than reopening this
5293    /// run's own `review_loop`: once any round in this run's history went
5294    /// clean, `review_conclusion` treats that as permanent by design (the
5295    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5296    /// way to force one more genuine reviewer wave out of *this* run without
5297    /// either rewriting history or weakening that guarantee for every other
5298    /// caller. A review-only run costs nothing extra — no implementation, no
5299    /// judging, no vote — and exercises the exact same review → verify →
5300    /// gate → (human) merge path, unmodified.
5301    pub async fn fix_selected(
5302        &mut self,
5303        ids: &[String],
5304        reason: &str,
5305        allow_stale: bool,
5306    ) -> Result<()> {
5307        let reason = reason.trim();
5308        if reason.is_empty() {
5309            bail!("a fix request needs a reason — that is the operator's own record of why");
5310        }
5311        if ids.is_empty() {
5312            bail!("no finding id given");
5313        }
5314        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5315            bail!(
5316                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5317                 has already concluded — can be given a targeted fix. A run still \
5318                 in progress should simply be resumed; a `merged` run's branch has \
5319                 already landed, so its answer is a fresh `magi review <branch>`, \
5320                 not reopening this run's own record",
5321                self.state.id,
5322                self.state.status.as_str()
5323            );
5324        }
5325        let Some(winner) = self.state.winner().cloned() else {
5326            bail!("run {} has no winning candidate to fix", self.state.id);
5327        };
5328        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5329            bail!(
5330                "branch `{}` no longer exists; this run cannot be extended",
5331                winner.branch
5332            );
5333        }
5334        let home = crate::run::home();
5335        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5336            bail!(
5337                "run {} is currently being worked on by another magi process",
5338                self.state.id
5339            );
5340        }
5341        // Held for the rest of this call, including the follow-up review
5342        // below: two `magi fix` invocations against the same run must not
5343        // both reach the worktree manipulation further down, which would
5344        // otherwise race to remove and recreate the same directory — see
5345        // [`FixClaim`]'s own doc.
5346        let _claim = FixClaim::acquire(&self.state.dir())?;
5347
5348        // Resolve every id before spending anything — an unknown id refuses
5349        // the whole request rather than silently dropping it — and dedup
5350        // while keeping the operator's own order.
5351        let mut seen = BTreeSet::new();
5352        let mut findings = Vec::new();
5353        let mut missing = Vec::new();
5354        for id in ids {
5355            if !seen.insert(id.clone()) {
5356                continue;
5357            }
5358            match self.state.finding(id) {
5359                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5360                    id: f.id.clone(),
5361                    severity: f.severity,
5362                    reviewer_vote: rec.vote,
5363                    round: round.round,
5364                    round_head: round.head.clone(),
5365                    reviewer: rec.reviewer,
5366                    agent: rec.agent.clone(),
5367                    file: f.file.clone(),
5368                    line: f.line,
5369                    title: f.title.clone(),
5370                    detail: f.detail.clone(),
5371                    outcome: OperatorFixOutcome::Pending,
5372                }),
5373                None => missing.push(id.clone()),
5374            }
5375        }
5376        if !missing.is_empty() {
5377            bail!(
5378                "unknown finding id(s): {}; nothing was changed",
5379                missing.join(", ")
5380            );
5381        }
5382
5383        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5384        let stale_details: Vec<(String, String)> = findings
5385            .iter()
5386            .filter(|f| f.round_head != head_at_request)
5387            .map(|f| (f.id.clone(), f.round_head.clone()))
5388            .collect();
5389        let stale = !stale_details.is_empty();
5390        if stale && !allow_stale {
5391            bail!(
5392                "the branch has moved since some finding(s) were raised — {} — now \
5393                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5394                stale_details
5395                    .iter()
5396                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5397                    .collect::<Vec<_>>()
5398                    .join(", "),
5399                short(&head_at_request)
5400            );
5401        }
5402
5403        let request = OperatorFixRequest {
5404            requested_at: Timestamp::now(),
5405            reason: reason.to_owned(),
5406            findings,
5407            head_at_request: head_at_request.clone(),
5408            allow_stale,
5409            stale,
5410            fix: None,
5411            result_head: None,
5412            follow_up_review_run: None,
5413        };
5414        self.state.event(
5415            "fix",
5416            format!(
5417                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5418                request.findings.len(),
5419                request
5420                    .findings
5421                    .iter()
5422                    .map(|f| f.id.as_str())
5423                    .collect::<Vec<_>>()
5424                    .join(", "),
5425            ),
5426        );
5427        // Recorded now, before any worktree work or the fixer call itself —
5428        // and re-saved at each checkpoint below: a crash at any point after
5429        // this (mid fixer call, mid follow-up review) must not lose the fact
5430        // that this was requested, for which findings, and why. Everything
5431        // past this point reads and writes through `request_index` rather
5432        // than a local variable, since `request` itself is moved here.
5433        self.state.operator_fixes.push(request);
5434        self.state.save()?;
5435        let request_index = self.state.operator_fixes.len() - 1;
5436
5437        // A fresh, dedicated worktree for this one call, never the winner's
5438        // own worktree in place: that one may already be gone (folded away),
5439        // and reusing it in place would leave the branch checked out there
5440        // when the follow-up review below tries to check it out again. Freed
5441        // immediately after, either way — but only once confirmed clean:
5442        // `worktree_remove` is a `git worktree remove --force`, which would
5443        // otherwise discard uncommitted work left there by the operator or
5444        // another process before this had a chance to even look at it.
5445        let repo = self.state.repo.clone();
5446        let coordination = self.coordinate(&repo).await?;
5447        if winner.worktree.exists() {
5448            // Lockfiles a rescue commit withheld stay untracked on purpose and
5449            // are already recorded; they are not the operator's work to protect.
5450            let dirty = git::git(
5451                &winner.worktree,
5452                &["status", "--porcelain", "--untracked-files=all"],
5453            )
5454            .await?;
5455            let only_withheld = dirty.lines().all(|l| {
5456                l.strip_prefix("?? ")
5457                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5458            });
5459            if !only_withheld {
5460                bail!(
5461                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5462                     discard them first",
5463                    winner.worktree.display()
5464                );
5465            }
5466            git::worktree_remove(&self.state.repo, &winner.worktree)
5467                .await
5468                .ok();
5469        }
5470        let fix_worktree = self.state.worktree_root().join("operator-fix");
5471        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5472        git::git(
5473            &self.state.repo,
5474            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5475        )
5476        .await
5477        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5478        if !git::is_clean(&fix_worktree).await? {
5479            git::worktree_remove(&self.state.repo, &fix_worktree)
5480                .await
5481                .ok();
5482            bail!(
5483                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5484                winner.branch
5485            );
5486        }
5487        let repo_for_setup = self.state.repo.clone();
5488        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5489            Ok(()) => {
5490                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5491            }
5492            Err(e) => Err(e),
5493        };
5494        if let Err(e) = setup {
5495            git::worktree_remove(&self.state.repo, &fix_worktree)
5496                .await
5497                .ok();
5498            return Err(e);
5499        }
5500
5501        let run_id = self.state.id.clone();
5502        let prompts = self.state.config.prompts.clone();
5503        let language = self.state.config.graph.language.clone();
5504        let sessions = self.state.config.graph.sessions;
5505        let artifacts = agent::artifacts_dir(&self.state.dir());
5506        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5507            .findings
5508            .iter()
5509            .map(|f| Finding {
5510                id: f.id.clone(),
5511                severity: f.severity,
5512                file: f.file.clone(),
5513                line: f.line,
5514                title: f.title.clone(),
5515                detail: f.detail.clone(),
5516            })
5517            .collect();
5518        let fix_prompt = prompt::operator_fix_in(
5519            &self.state.instruction,
5520            &finding_list,
5521            reason,
5522            &stale_details,
5523            &head_at_request,
5524            &language,
5525            &self.state.config.graph.github_language,
5526        );
5527        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5528        drop(coordination);
5529        let (job, seat, out) = self
5530            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5531                prompt: fix_prompt.clone(),
5532                spec,
5533                seat,
5534                cwd: fix_worktree.clone(),
5535                timeout,
5536                allow_write: true,
5537                sessions,
5538                artifacts: artifacts.clone(),
5539                stem: "operator-fix".to_owned(),
5540                handover: None,
5541            })
5542            .await;
5543        let agent_id = seat.agent.clone();
5544
5545        let mut fix = FixRecord {
5546            agent: agent_id,
5547            addressed: Vec::new(),
5548            rejected: Vec::new(),
5549            notes: String::new(),
5550            committed: false,
5551            failed: None,
5552            duration_ms: 0,
5553            continuation: None,
5554        };
5555        let mut final_seat = seat.clone();
5556        match out {
5557            AgentOutcome::Ok(o) => {
5558                fix.duration_ms = o.duration_ms;
5559                let parsed = verdict::extract_json::<FixReport>(&o.text);
5560                let incomplete_reason = match &parsed {
5561                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5562                        "the reply parsed, but it reported a command whose own CLI \
5563                         never confirmed an exit status"
5564                            .to_owned(),
5565                    ),
5566                    Ok(_) => None,
5567                    Err(e) => Some(e.to_string()),
5568                };
5569                match incomplete_reason {
5570                    None => {
5571                        let report = parsed.expect("checked Ok above");
5572                        fix.addressed = report.addressed;
5573                        fix.rejected = report.rejected;
5574                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5575                    }
5576                    Some(reason) => {
5577                        let (resumed_seat, resolved, failure, cont) = self
5578                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5579                            .await;
5580                        fix.duration_ms += cont.cumulative_wait_ms;
5581                        fix.continuation = Some(cont);
5582                        final_seat = resumed_seat;
5583                        match resolved {
5584                            Some(report) => {
5585                                fix.addressed = report.addressed;
5586                                fix.rejected = report.rejected;
5587                                fix.notes =
5588                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5589                            }
5590                            None => fix.failed = failure,
5591                        }
5592                    }
5593                }
5594            }
5595            AgentOutcome::Dropped(o) => {
5596                fix.duration_ms = o.duration_ms;
5597                let why = o
5598                    .dropped
5599                    .as_ref()
5600                    .map(|d| d.why.as_str())
5601                    .unwrap_or("the CLI ended the stream without delivering its answer");
5602                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5603            }
5604            AgentOutcome::Quota(o) => {
5605                self.state.quota.push(QuotaLoss {
5606                    seat: final_seat.key.clone(),
5607                    node: "fix".to_owned(),
5608                    at: Timestamp::now(),
5609                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5610                });
5611                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5612            }
5613            AgentOutcome::Failed(e) => fix.failed = Some(e),
5614            AgentOutcome::ResourceBlocked(e) => {
5615                fix.failed = Some(format!("the fixer was not started: {e}"));
5616            }
5617        }
5618        if fix.continuation.is_none() {
5619            fix.continuation = Some(ContinuationRecord::not_needed());
5620        }
5621        self.state.seats.insert(final_seat.key.clone(), final_seat);
5622
5623        let rescue_message = format!(
5624            "magi: operator-selected fix ({}) (uncommitted work)",
5625            self.state.operator_fixes[request_index]
5626                .findings
5627                .iter()
5628                .map(|f| f.id.as_str())
5629                .collect::<Vec<_>>()
5630                .join(", ")
5631        );
5632        let repo = self.state.repo.clone();
5633        let coordination = self.coordinate(&repo).await?;
5634        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5635            self.state.note_withheld("fix", &r.withheld);
5636        }
5637        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5638        fix.committed = after != head_at_request;
5639        git::worktree_remove(&self.state.repo, &fix_worktree)
5640            .await
5641            .ok();
5642        drop(coordination);
5643
5644        self.state.event(
5645            "fix",
5646            match &fix.failed {
5647                Some(reason) => format!(
5648                    "operator fix: adoption report was lost ({reason}); {}",
5649                    if fix.committed {
5650                        "committed"
5651                    } else {
5652                        "NO new commit"
5653                    }
5654                ),
5655                None => format!(
5656                    "operator fix: {} addressed, {} rejected, {}",
5657                    fix.addressed.len(),
5658                    fix.rejected.len(),
5659                    if fix.committed {
5660                        "committed"
5661                    } else {
5662                        "NO new commit"
5663                    }
5664                ),
5665            },
5666        );
5667
5668        // Every selected finding gets an outcome — never left `Pending` once
5669        // the fixer's own turn is over. A report that never came back at all
5670        // marks every one of them `Unreported`, not silently "not addressed":
5671        // quota, a dropped stream, or an exhausted continuation are gaps in
5672        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5673        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5674        for f in &mut self.state.operator_fixes[request_index].findings {
5675            f.outcome = if fix.failed.is_some() {
5676                OperatorFixOutcome::Unreported
5677            } else if fix.addressed.contains(&f.id) {
5678                OperatorFixOutcome::Addressed
5679            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5680                OperatorFixOutcome::Rejected { why: r.why.clone() }
5681            } else {
5682                OperatorFixOutcome::Unreported
5683            };
5684        }
5685
5686        let committed = fix.committed;
5687        if committed {
5688            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5689        }
5690        self.state.operator_fixes[request_index].fix = Some(fix);
5691        // Saved again now that the fixer's own outcome is final, on top of
5692        // the save right after the request was first pushed above.
5693        self.state.save()?;
5694
5695        if committed {
5696            self.state.event(
5697                "fix",
5698                format!(
5699                    "operator fix committed {}; opening a follow-up review-only run",
5700                    short(&after)
5701                ),
5702            );
5703            // The operator asked for the fix, and the follow-up serves whatever
5704            // task the run it follows served.
5705            let origin =
5706                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5707            match Self::review(
5708                &self.state.repo,
5709                &winner.branch,
5710                self.state.config.clone(),
5711                origin,
5712            )
5713            .await
5714            {
5715                Ok(mut follow_up) => {
5716                    follow_up.state.event(
5717                        "start",
5718                        format!(
5719                            "requested by an operator fix on run {} for finding(s) {}",
5720                            self.state.id,
5721                            self.state.operator_fixes[request_index]
5722                                .findings
5723                                .iter()
5724                                .map(|f| f.id.as_str())
5725                                .collect::<Vec<_>>()
5726                                .join(", "),
5727                        ),
5728                    );
5729                    follow_up.state.save()?;
5730                    let follow_up_id = follow_up.state.id.clone();
5731                    // The follow-up is a run like any other: it belongs to the
5732                    // task of the run it follows, or to one filed for it.
5733                    let adopted = match crate::direct::adopt(
5734                        &follow_up.state,
5735                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5736                    ) {
5737                        Ok(a) => a,
5738                        Err(e) => {
5739                            // An ownerless run must not spend agent calls; it
5740                            // stays saved, and `magi run --resume` adopts it.
5741                            self.state.event(
5742                                "fix",
5743                                format!(
5744                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5745                                ),
5746                            );
5747                            self.state.save()?;
5748                            return Ok(());
5749                        }
5750                    };
5751                    let executed = follow_up.execute().await;
5752                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5753                    if let Some(a) = adopted {
5754                        a.finish(&follow_up.state, executed);
5755                    }
5756                    if let Some(e) = failure {
5757                        self.state.event(
5758                            "fix",
5759                            format!(
5760                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5761                            ),
5762                        );
5763                    }
5764                    self.state.operator_fixes[request_index].follow_up_review_run =
5765                        Some(follow_up_id);
5766                }
5767                Err(e) => {
5768                    self.state.event(
5769                        "fix",
5770                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5771                    );
5772                }
5773            }
5774            self.state.save()?;
5775        }
5776
5777        Ok(())
5778    }
5779
5780    // --------------------------------------------------------------- review
5781
5782    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5783    /// names, in order, each at most once, moving on only when the call
5784    /// advances (an error, a quota hit or nothing usable - the same decision
5785    /// point `agent::chain_advances` is for the other chained roles).
5786    ///
5787    /// Each agent gets its own seat from `Runner::seat`: the same agent
5788    /// continues its conversation (the winner's own implementer seat when it
5789    /// is the winner's author, now that the competition is over), another
5790    /// takes a fresh one so the full prompt is sent again. A handover is
5791    /// recorded under `node`, which is what makes the fallback stick for the
5792    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5793    /// is returned, so an exhausted chain reads exactly like a single failed
5794    /// fixer: one quota loss, the same wording, the same refund. An earlier
5795    /// attempt's edits are left in the tree and judged, with the final
5796    /// attempt's, by what git says afterwards.
5797    ///
5798    /// Returns the job that produced the outcome, for `continue_fix_report`.
5799    async fn ask_fixer(
5800        &mut self,
5801        winner: &Candidate,
5802        node: &'static str,
5803        round: Option<usize>,
5804        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5805    ) -> (SeatJob, SeatState, AgentOutcome) {
5806        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5807        self.ask_fixer_among(attempts, node, round, None, build)
5808            .await
5809    }
5810
5811    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5812    /// worktree and a commit: before the next agent is asked after one that
5813    /// did not satisfy the call, the worktree is put back on that commit, so
5814    /// a failed agent's partial edits are not carried into the next one.
5815    async fn ask_fixer_among(
5816        &mut self,
5817        attempts: Vec<(AgentSpec, String)>,
5818        node: &'static str,
5819        round: Option<usize>,
5820        restore: Option<(&Path, &str)>,
5821        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5822    ) -> (SeatJob, SeatState, AgentOutcome) {
5823        let run_id = self.state.id.clone();
5824        let prompts = self.state.config.prompts.clone();
5825        let cache = self.state.config.cache_dir();
5826        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5827        let mut last = None;
5828        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5829            let seat = self.seat(&key, &spec.id);
5830            let mut job = build(spec.clone(), seat);
5831            if i > 0 {
5832                job.stem = format!("{}-{}", job.stem, spec.id);
5833            }
5834            let ctx = WaveCtx {
5835                carry_seats: false,
5836                pauses: Some((&self.pause, &self.interrupt)),
5837                run: &run_id,
5838                node,
5839                prompts: &prompts,
5840                cache: cache.as_deref(),
5841                round,
5842            };
5843            let (seat, out) =
5844                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5845            // A resource block never advances the chain: the chosen agent
5846            // was not asked, so it must be the one asked again on resume.
5847            let advances = match &out {
5848                AgentOutcome::Ok(o) => agent::output_advances(o),
5849                AgentOutcome::ResourceBlocked(_) => false,
5850                _ => true,
5851            };
5852            if let Some(next) = ids.get(i + 1)
5853                && advances
5854            {
5855                self.state.seats.insert(seat.key.clone(), seat.clone());
5856                let class =
5857                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5858                let reason = match &out {
5859                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5860                    other => fail_reason(other),
5861                };
5862                // A failed agent's edits must not reach the next one, and a
5863                // restore that did not work ends the chain: the call fails
5864                // and the caller's own (confirmed) restore decides.
5865                if let Some((wt, rev)) = restore
5866                    && let Err(e) = self.graft_restore(wt, rev).await
5867                {
5868                    let why =
5869                        format!("could not restore the worktree before the next fixer: {e:#}");
5870                    tracing::warn!("{why}");
5871                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5872                        AgentOutcome::ResourceBlocked(why)
5873                    } else {
5874                        AgentOutcome::Failed(why)
5875                    };
5876                    last = Some((job, seat, out));
5877                    break;
5878                }
5879                record_handover(
5880                    &mut self.state,
5881                    node,
5882                    &seat.key,
5883                    &spec.id,
5884                    next,
5885                    &class,
5886                    &reason,
5887                );
5888                continue;
5889            }
5890            last = Some((job, seat, out));
5891            break;
5892        }
5893        last.expect("the fixer chain always has an entry")
5894    }
5895
5896    /// The one repair a verification-only failure earns (see
5897    /// [`crate::recovery`]), before the first review round of a run that took
5898    /// a branch over.
5899    ///
5900    /// Saved as started *before* the fixer is called, so a resume never
5901    /// repairs twice; a start found on re-entry is rolled back to the commit
5902    /// it began from. A failed or unusable fixer leaves the tree as it was and
5903    /// the review goes on over it, so the first round's own verification
5904    /// reports the failure again. The failure is never carried into a round as
5905    /// a result: verification is re-run on whatever commit this leaves.
5906    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5907        use crate::recovery::{Mode, Repair};
5908        if !self.state.reviews.is_empty() {
5909            return Ok(());
5910        }
5911        let Some(mut rec) = self.state.recovery.clone() else {
5912            return Ok(());
5913        };
5914        if rec.mode != Mode::VerifyRepair {
5915            return Ok(());
5916        }
5917        let wt = winner.worktree.clone();
5918        let mut head = git::rev_parse(&wt, "HEAD").await?;
5919        match rec.repair.clone() {
5920            Repair::Done { .. } => return Ok(()),
5921            Repair::WaitingForRestore { before } => {
5922                self.graft_restore(&wt, &before).await?;
5923                head = before;
5924                rec.repair = Repair::NotStarted;
5925                self.state.recovery = Some(rec.clone());
5926                self.state.save()?;
5927            }
5928            Repair::Started { before } => {
5929                // Interrupted: the unfinished edits are not trusted.
5930                // An unconfirmed rollback blocks the run with `Started` kept,
5931                // so a resume retries it; the tree is never reviewed as is.
5932                if let Err(e) = self.graft_restore(&wt, &before).await {
5933                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5934                        return Err(e);
5935                    }
5936                    return self
5937                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5938                        .await;
5939                }
5940                rec.repair = Repair::Done {
5941                    before: before.clone(),
5942                    after: before,
5943                    committed: false,
5944                    failed: Some("interrupted; rolled back".to_owned()),
5945                };
5946                self.state
5947                    .event("recovery", "the repair was interrupted and rolled back");
5948                self.state.recovery = Some(rec);
5949                self.state.save()?;
5950                return Ok(());
5951            }
5952            Repair::NotStarted => {}
5953        }
5954        if crate::recovery::recheck(&mut rec, &head) {
5955            self.state.event("recovery", rec.summary());
5956            self.state.recovery = Some(rec);
5957            self.state.save()?;
5958            return Ok(());
5959        }
5960        rec.repair = Repair::Started {
5961            before: head.clone(),
5962        };
5963        self.state.recovery = Some(rec.clone());
5964        self.state.save()?;
5965
5966        let language = self.state.config.graph.language.clone();
5967        let sessions = self.state.config.graph.sessions;
5968        let artifacts = agent::artifacts_dir(&self.state.dir());
5969        let fix_prompt = prompt::recovery_fix(
5970            &self.state.instruction,
5971            &rec,
5972            self.state.config.graph.review_rounds,
5973            &language,
5974        );
5975        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5976        self.state.event(
5977            "recovery",
5978            format!(
5979                "repair: the fixer starts from the failed verification on {}",
5980                short(&head)
5981            ),
5982        );
5983        let (_job, seat, out) = self
5984            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5985                prompt: fix_prompt.clone(),
5986                spec,
5987                seat,
5988                cwd: wt.clone(),
5989                timeout,
5990                allow_write: true,
5991                sessions,
5992                artifacts: artifacts.clone(),
5993                stem: "fix-recovery".to_owned(),
5994                handover: None,
5995            })
5996            .await;
5997        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5998            rec.repair = Repair::WaitingForRestore { before: head };
5999            self.state.recovery = Some(rec);
6000            return self.stop_for_resource();
6001        }
6002        let failed = match &out {
6003            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6004            AgentOutcome::Quota(o) => {
6005                self.state.quota.push(QuotaLoss {
6006                    seat: seat.key.clone(),
6007                    node: "fix".to_owned(),
6008                    at: Timestamp::now(),
6009                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6010                });
6011                Some("rate limited (quota); the fixer could not run".to_owned())
6012            }
6013            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6014            AgentOutcome::Failed(e) => Some(e.clone()),
6015        };
6016        self.state.seats.insert(seat.key.clone(), seat);
6017        let coordination = self.coordinate(&wt).await?;
6018        let failed = match failed {
6019            // The agent's partial edits are not kept.
6020            Some(why) => {
6021                if let Err(e) = git::restore_to(&wt, &head).await {
6022                    return self
6023                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6024                        .await;
6025                }
6026                Some(why)
6027            }
6028            None => {
6029                if let Ok(r) = git::rescue_commit(
6030                    &wt,
6031                    "magi: recovery repair of the failed verification (uncommitted work)",
6032                )
6033                .await
6034                {
6035                    self.state.note_withheld("fix", &r.withheld);
6036                }
6037                None
6038            }
6039        };
6040        let after = git::rev_parse(&wt, "HEAD").await?;
6041        drop(coordination);
6042        self.state.event(
6043            "recovery",
6044            match &failed {
6045                Some(why) => format!("repair: nothing kept ({why})"),
6046                None if after != head => {
6047                    format!("repair: committed {}", short(&after))
6048                }
6049                None => "repair: the fixer made no commit".to_owned(),
6050            },
6051        );
6052        rec.repair = Repair::Done {
6053            before: head.clone(),
6054            committed: after != head,
6055            after,
6056            failed,
6057        };
6058        self.state.recovery = Some(rec);
6059        self.state.save()?;
6060        Ok(())
6061    }
6062
6063    /// A repair whose rollback could not be confirmed: the run is blocked and
6064    /// resumable, and the unconfirmed state is never reviewed.
6065    async fn stop_recovery(
6066        &mut self,
6067        mut rec: crate::recovery::RecoveryRecord,
6068        head: &str,
6069        why: String,
6070    ) -> Result<()> {
6071        // `Started` stays recorded: a resume retries the restore.
6072        rec.repair = crate::recovery::Repair::Started {
6073            before: head.to_owned(),
6074        };
6075        self.state.recovery = Some(rec);
6076        self.state
6077            .event("recovery", format!("repair could not be undone: {why}"));
6078        self.state.status = RunStatus::Blocked;
6079        self.state.save()?;
6080        bail!("recovery repair could not be rolled back: {why}")
6081    }
6082
6083    async fn review_loop(&mut self) -> Result<()> {
6084        if self.hold_base_sync().await? {
6085            return Ok(());
6086        }
6087        loop {
6088            if self
6089                .state
6090                .base_sync
6091                .as_ref()
6092                .is_some_and(|s| s.conflict.is_some())
6093            {
6094                return Ok(());
6095            }
6096            if self.catchup_repairable() {
6097                let record = self.state.reviews.last().expect("repairable round").clone();
6098                let winner = self
6099                    .state
6100                    .winner()
6101                    .cloned()
6102                    .context("catch-up repair has no winner")?;
6103                if !self.fix_review_round(&winner, record).await? {
6104                    return Ok(());
6105                }
6106            }
6107            self.review_loop_pass().await?;
6108            if !self.catchup_repairable() {
6109                return Ok(());
6110            }
6111        }
6112    }
6113
6114    /// Derive an owed catch-up repair from durable round evidence, including
6115    /// reentry after a crash before the fixer was dispatched.
6116    fn catchup_repairable(&self) -> bool {
6117        self.state.reviews.len() < self.state.config.graph.review_rounds
6118            && self.state.reviews.last().is_some_and(|r| {
6119                !r.clean
6120                    && r.blocking == 0
6121                    && r.fix.is_none()
6122                    && r.e2e_status() == E2eStatus::Failed
6123            })
6124    }
6125
6126    async fn review_loop_pass(&mut self) -> Result<()> {
6127        // A base that would not rebase is a person's decision, not a review
6128        // round: nothing here would change the answer, and reviewers and a
6129        // fixer would be spending real budget on a tree that cannot land
6130        // regardless of what they find.
6131        if self
6132            .state
6133            .base_sync
6134            .as_ref()
6135            .is_some_and(|s| s.conflict.is_some())
6136        {
6137            return Ok(());
6138        }
6139        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6140        // agent files with `magi task add` name the run that paid for it. The
6141        // prompt overlay is cloned alongside it because the waves borrow it
6142        // while `self` is mutably borrowed by the node's own bookkeeping.
6143        let run_id = self.state.id.clone();
6144        let prompts = self.state.config.prompts.clone();
6145        let Some(winner) = self.state.winner().cloned() else {
6146            return Ok(());
6147        };
6148        let max_rounds = self.state.config.graph.review_rounds;
6149        // A clean round, an exhausted round budget, or a stalled tree (see
6150        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6151        // they are recorded — recomputed here, not read off `status`, so a
6152        // reentry into a run that already stopped restates the identical
6153        // verdict instead of silently handing back whatever an earlier node
6154        // in this same walk clobbered `status` to (a solo-candidate
6155        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6156        // below runs an empty range once the budget is spent, and would
6157        // otherwise fall through without touching `status` at all.
6158        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6159            // A reentry after a crash between the last round's save and
6160            // `stop_reviewing` reaches the hand-off here, not there.
6161            if status == RunStatus::Gating {
6162                self.record_contested_handoff();
6163            }
6164            self.state.status = status;
6165            self.state.save()?;
6166            return Ok(());
6167        }
6168        self.state.status = RunStatus::Reviewing;
6169        // A last recorded round whose own verification never resolved
6170        // (`ResourceBlocked` — the shared build cache, not the patch) is
6171        // never a concluded round, whatever the round budget says: starting
6172        // a fresh round on top of it would spend a whole new reviewer wave
6173        // re-reading an unchanged patch instead of just retrying the one
6174        // check that actually needs it, and once the budget is spent the
6175        // loop below has nothing left to do at all (its range is empty).
6176        // Retry that check directly instead, exactly the same retry
6177        // `stop_reviewing` already does for its own catch-up case.
6178        if self.state.reviews.last().is_some_and(|r| {
6179            r.e2e_status() == E2eStatus::ResourceBlocked
6180                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6181        }) {
6182            let shell = self.state.config.shell();
6183            return self
6184                .stop_unresolved_review(
6185                    "the last round's own verification never resolved",
6186                    &shell,
6187                    &winner.worktree,
6188                )
6189                .await;
6190        }
6191
6192        self.recovery_repair(&winner).await?;
6193        if self.resource_halt()? {
6194            return Ok(());
6195        }
6196
6197        let repo = self.state.repo.clone();
6198        let root = self.state.worktree_root();
6199        let language = self.state.config.graph.language.clone();
6200        let sessions = self.state.config.graph.sessions;
6201        let artifacts = self.merge_repair_artifacts();
6202        let base = self.landing_base();
6203        let base_short = short(&base);
6204        let reviewers = self.roles.reviewers.clone();
6205        let shell = self.state.config.shell();
6206
6207        for round in (self.state.reviews.len() + 1)..=max_rounds {
6208            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6209            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6210            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6211            // The prior round's own record, already persisted — never a
6212            // hand-carried variable of just its failing output: that is
6213            // exactly what let a round's e2e result drift out of sync with
6214            // which commit it was actually about (see `SCHEMA`'s doc for
6215            // schema 8). Judged against `head`, the commit reviewers are
6216            // about to look at now, so the summary always reads as "an
6217            // earlier head" here — this round's own patch has not been
6218            // checked yet.
6219            let prev_verification = self
6220                .state
6221                .reviews
6222                .last()
6223                .and_then(|r| r.verification_summary(&head));
6224            // Only the first round of a run that took a branch over: later
6225            // rounds have this run's own history.
6226            let prior_section = match (&self.state.recovery, round) {
6227                (Some(rec), 1) if self.state.reviews.is_empty() => {
6228                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6229                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6230                            Some(crate::recovery::head_of(
6231                                &git::diff(&winner.worktree, reviewed, "HEAD")
6232                                    .await
6233                                    .unwrap_or_default(),
6234                                crate::recovery::SINCE_DIFF_MAX,
6235                            ))
6236                        }
6237                        _ => None,
6238                    };
6239                    Some(prompt::recovery_review(rec, since.as_deref()))
6240                }
6241                _ => None,
6242            };
6243
6244            // Each reviewer gets its own detached checkout of exactly this
6245            // commit: nobody can perturb the winner's tree, and the fixer can
6246            // keep working without racing a reviewer.
6247            let mut jobs = Vec::new();
6248            let coordination = self.coordinate(&repo).await?;
6249            for (r, spec) in reviewers.iter().cloned().enumerate() {
6250                let wt = root.join(format!("review-{}", r + 1));
6251                // `clean -fdx` in a reset wipes setup's products, so every
6252                // round sets the seat up again.
6253                if wt.exists() {
6254                    git::reset_detached(&wt, &head).await?;
6255                } else {
6256                    git::worktree_add_detached(&repo, &wt, &head).await?;
6257                }
6258                self.setup_seat_worktree(&repo, &wt, false).await?;
6259                let seat_key = format!("review-{}", r + 1);
6260                // The seat starts the round on whoever answered it last, not
6261                // on the agent the spec names, so a failure is not re-paid.
6262                let spec = pick_start_spec(
6263                    &self.roles.reviewer_roster,
6264                    spec,
6265                    self.state.seat_history.get(&seat_key),
6266                );
6267                let seat = self.seat(&seat_key, &spec.id);
6268                jobs.push(SeatJob {
6269                    prompt: prompt::review(&prompt::ReviewCtx {
6270                        instruction: &self.state.instruction,
6271                        branch: &winner.branch,
6272                        base_short: &base_short,
6273                        stat: &stat,
6274                        patch: &patch,
6275                        verification: prev_verification.as_ref(),
6276                        prior: prior_section.as_deref(),
6277                        reviewers: reviewers.len(),
6278                        round,
6279                        rounds: max_rounds,
6280                        // A review-only run has no rankings, so nothing
6281                        // competed for this patch and the reviewer is told so.
6282                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6283                        lens: Lens::for_seat(r),
6284                        language: &language,
6285                        github_language: &self.state.config.graph.github_language,
6286                    }),
6287                    spec,
6288                    seat,
6289                    cwd: wt,
6290                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6291                    allow_write: false,
6292                    sessions,
6293                    artifacts: artifacts.clone(),
6294                    stem: format!("review-{round}-{}", r + 1),
6295                    handover: None,
6296                });
6297            }
6298
6299            drop(coordination);
6300            self.state.event(
6301                "review",
6302                format!(
6303                    "round {round}: {} reviewers on {}",
6304                    jobs.len(),
6305                    short(&head)
6306                ),
6307            );
6308            let mut quota_losses = Vec::new();
6309            let review_retries = self.state.config.graph.retries;
6310            let review_cache = self.state.config.cache_dir();
6311            let ctx = WaveCtx {
6312                carry_seats: true,
6313                pauses: Some((&self.pause, &self.interrupt)),
6314                run: &run_id,
6315                node: "review",
6316                prompts: &prompts,
6317                cache: review_cache.as_deref(),
6318                round: Some(round),
6319            };
6320            let results = ask_json_wave::<Review>(
6321                jobs,
6322                Arc::clone(&self.sem),
6323                review_retries,
6324                &self.roles.reviewer_roster,
6325                &ctx,
6326                &mut quota_losses,
6327                &mut self.state,
6328                &|_: &Review| Ok(()),
6329            )
6330            .await;
6331            if self.state.resource_block.is_some() {
6332                return self.stop_for_resource();
6333            }
6334            // Counted before the move below: how many of *this* round's
6335            // reviewer seats were lost to their own rate limit, as opposed to
6336            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6337            let round_quota_missing = quota_losses.len();
6338            self.state.quota.extend(quota_losses);
6339
6340            let mut records = Vec::new();
6341            let mut all_findings = Vec::new();
6342            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6343                let agent_id = seat.agent.clone();
6344                self.state.seats.insert(seat.key.clone(), seat);
6345                let mut record = ReviewRecord {
6346                    reviewer: r + 1,
6347                    agent: agent_id,
6348                    summary: String::new(),
6349                    findings: Vec::new(),
6350                    vote: None,
6351                    failed: None,
6352                    duration_ms: 0,
6353                    // Set for both outcomes: `failed: Some(_)` with
6354                    // `attempts > 0` is a seat every retry still lost, not a
6355                    // recovered one — only `failed: None` with `attempts > 0`
6356                    // reads as "answered after a nudge" (see this field's own
6357                    // doc).
6358                    attempts,
6359                };
6360                match res {
6361                    Ok((review, out)) => {
6362                        // Sanitized here, at the point every other piece of
6363                        // agent prose in this file is (candidate summaries,
6364                        // deliberation turns, vote reasons): a reviewer's own
6365                        // words are the one thing about it that could name
6366                        // it, and reconsideration below broadcasts this same
6367                        // summary and these same findings to every other
6368                        // seat on the panel.
6369                        record.summary =
6370                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6371                        record.vote = Some(review.vote);
6372                        record.duration_ms = out.duration_ms;
6373                        for (n, mut f) in review.findings.into_iter().enumerate() {
6374                            // ids are magi's, never the agent's: the fixer's
6375                            // adoption report is keyed by them.
6376                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6377                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6378                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6379                            // `file` is agent-supplied prose too, never
6380                            // checked against the real tree — the same
6381                            // exposure `title`/`detail` above have, just in
6382                            // a field easy to forget because it looks like a
6383                            // path rather than free text.
6384                            f.file = f
6385                                .file
6386                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6387                            all_findings.push(f.clone());
6388                            record.findings.push(f);
6389                        }
6390                        self.state.event(
6391                            "review",
6392                            format!(
6393                                "round {round}: reviewer {} voted {} with {} finding(s)",
6394                                r + 1,
6395                                review.vote.label(),
6396                                record.findings.len()
6397                            ),
6398                        );
6399                    }
6400                    Err(e) => {
6401                        record.failed = Some(e.to_string());
6402                        self.state.event(
6403                            "review",
6404                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6405                        );
6406                    }
6407                }
6408                records.push(record);
6409            }
6410
6411            // Tally the round's votes and, if they split, spend the one
6412            // round of reconsideration the split -> deliberate -> revote
6413            // shape `judge`/`vote` use for the panel, sized down to what a
6414            // read-only review round can afford: one round, and a revote
6415            // rather than an argument, because the panel already wrote its
6416            // reasoning down as findings the first time around.
6417            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6418            let vote_split =
6419                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6420            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6421            if vote_split {
6422                self.state.event(
6423                    "review",
6424                    format!(
6425                        "round {round}: votes split ({}) — one round of reconsideration",
6426                        initial_votes
6427                            .iter()
6428                            .map(|v| v.label())
6429                            .collect::<Vec<_>>()
6430                            .join(", ")
6431                    ),
6432                );
6433                // Seats read every seat's findings and votes, still numbered
6434                // and never named — the same anonymity `review` itself keeps.
6435                let panel: Vec<ReviewSeatReport<'_>> = records
6436                    .iter()
6437                    .filter_map(|r| {
6438                        r.vote.map(|vote| ReviewSeatReport {
6439                            reviewer: r.reviewer,
6440                            vote,
6441                            summary: &r.summary,
6442                            findings: &r.findings,
6443                        })
6444                    })
6445                    .collect();
6446
6447                let mut jobs = Vec::new();
6448                let mut seats_at = Vec::new();
6449                for (r, spec) in reviewers.iter().cloned().enumerate() {
6450                    // A seat with no initial vote has nothing to reconsider
6451                    // from and stays absent, the same as it stayed absent
6452                    // from `panel` above.
6453                    if records[r].vote.is_none() {
6454                        continue;
6455                    }
6456                    let wt = root.join(format!("review-{}", r + 1));
6457                    let seat_key = format!("review-{}", r + 1);
6458                    let spec = self.occupant(&seat_key, spec);
6459                    let seat = self.seat(&seat_key, &spec.id);
6460                    // A seat with no live session has already forgotten the
6461                    // initial review's prompt — restate the patch it is
6462                    // voting on, the same as `deliberate`/`vote` do for a
6463                    // judge in the same position.
6464                    // The panel already carries this seat's own review and
6465                    // vote, so restating the patch makes the prompt whole for
6466                    // a seat handed to another agent.
6467                    let build = |with_patch: bool| {
6468                        prompt::review_reconsider(&ReviewReconsiderCtx {
6469                            instruction: &self.state.instruction,
6470                            reviewer: r + 1,
6471                            lens: Lens::for_seat(r),
6472                            panel: &panel,
6473                            patch: with_patch.then_some(ReviewPatch {
6474                                branch: &winner.branch,
6475                                base_short: &base_short,
6476                                stat: &stat,
6477                                patch: &patch,
6478                            }),
6479                            round,
6480                            rounds: max_rounds,
6481                            language: &language,
6482                        })
6483                    };
6484                    let full = build(true);
6485                    let prompt = if has_context(&spec, &seat, sessions) {
6486                        build(false)
6487                    } else {
6488                        full.clone()
6489                    };
6490                    jobs.push(SeatJob {
6491                        prompt,
6492                        spec,
6493                        seat,
6494                        cwd: wt,
6495                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6496                        allow_write: false,
6497                        sessions,
6498                        artifacts: artifacts.clone(),
6499                        stem: format!("review-{round}-reconsider-{}", r + 1),
6500                        handover: Some(full),
6501                    });
6502                    seats_at.push(r);
6503                }
6504
6505                let mut recon_quota_losses = Vec::new();
6506                let recon_cache = self.state.config.cache_dir();
6507                let recon_ctx = WaveCtx {
6508                    carry_seats: true,
6509                    pauses: Some((&self.pause, &self.interrupt)),
6510                    run: &run_id,
6511                    node: "review",
6512                    prompts: &prompts,
6513                    cache: recon_cache.as_deref(),
6514                    round: Some(round),
6515                };
6516                let recon_results = ask_json_wave::<ReviewRevote>(
6517                    jobs,
6518                    Arc::clone(&self.sem),
6519                    review_retries,
6520                    &self.roles.reviewer_roster,
6521                    &recon_ctx,
6522                    &mut recon_quota_losses,
6523                    &mut self.state,
6524                    &|_: &ReviewRevote| Ok(()),
6525                )
6526                .await;
6527                if self.state.resource_block.is_some() {
6528                    return self.stop_for_resource();
6529                }
6530                self.state.quota.extend(recon_quota_losses);
6531
6532                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6533                    let agent_id = seat.agent.clone();
6534                    self.state.seats.insert(seat.key.clone(), seat);
6535                    let mut rec = ReviewRevoteRecord {
6536                        reviewer: r + 1,
6537                        agent: agent_id,
6538                        vote: None,
6539                        reason: String::new(),
6540                        failed: None,
6541                    };
6542                    match res {
6543                        Ok((rv, _)) => {
6544                            rec.vote = Some(rv.vote);
6545                            rec.reason =
6546                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6547                            self.state.event(
6548                                "review",
6549                                format!(
6550                                    "round {round}: reviewer {} revoted {}",
6551                                    r + 1,
6552                                    rv.vote.label()
6553                                ),
6554                            );
6555                        }
6556                        Err(e) => {
6557                            rec.failed = Some(e.to_string());
6558                            self.state.event(
6559                                "review",
6560                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6561                            );
6562                        }
6563                    }
6564                    reconsideration.push(rec);
6565                }
6566            } else if initial_votes.len() > 1 {
6567                self.state.event(
6568                    "review",
6569                    format!(
6570                        "round {round}: votes agreed ({}) — no reconsideration",
6571                        initial_votes[0].label()
6572                    ),
6573                );
6574            }
6575
6576            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6577            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6578            // A round that already has a blocking finding and a round left to
6579            // try is going back to the fixer no matter what `verify.e2e`
6580            // says, so running it first only spends the loop's slowest step
6581            // (minutes, for a Rust repo's full test suite) on a head about
6582            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6583            // runs once a round has no blocking findings left (see
6584            // `round_is_clean`, which a deferred — empty — `e2e` can never
6585            // satisfy since `blocking` is nonzero whenever this branch is
6586            // taken), and `stop_reviewing` forces a real run before it will
6587            // ever read a deferred round as green.
6588            let defer_e2e =
6589                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6590            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6591                let reason =
6592                    format!("{blocking} blocking finding(s) already required a fix this round");
6593                self.state.event(
6594                    "verify",
6595                    format!(
6596                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6597                         {}); it will run once a round has none left",
6598                        short(&head)
6599                    ),
6600                );
6601                (Vec::new(), false, true, Some(reason))
6602            } else {
6603                let e2e_commands = self.state.config.verify.e2e.clone();
6604                let cache_dir = self.state.config.cache_dir();
6605                let context = format!("round {round}");
6606                let wait = self.repository_wait();
6607                let (e2e, verify_retried) = with_cache_lease(
6608                    &mut self.state,
6609                    cache_dir.as_deref(),
6610                    "e2e",
6611                    "e2e",
6612                    &winner.worktree,
6613                    &head,
6614                    verify_timeout,
6615                    &context,
6616                    &wait,
6617                    |state, budget| {
6618                        let shell = shell.clone();
6619                        let e2e_commands = e2e_commands.clone();
6620                        let worktree = winner.worktree.clone();
6621                        let context = context.clone();
6622                        async move {
6623                            run_e2e_with_retry(
6624                                state,
6625                                &shell,
6626                                &e2e_commands,
6627                                &worktree,
6628                                budget,
6629                                &context,
6630                            )
6631                            .await
6632                        }
6633                    },
6634                )
6635                .await?;
6636                (e2e, verify_retried, false, None)
6637            };
6638
6639            let expected = records.len();
6640            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6641            let incomplete = answered < expected;
6642            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6643            let policy = self.state.config.graph.incomplete_review;
6644            let clean = round_is_clean(
6645                blocking,
6646                e2e_ok,
6647                answered,
6648                expected,
6649                round_quota_missing,
6650                policy,
6651            );
6652
6653            let mut round_record = ReviewRound {
6654                round,
6655                head: head.clone(),
6656                verified_head: None,
6657                verified_at: None,
6658                reviews: records,
6659                e2e,
6660                verify_retried,
6661                e2e_deferred,
6662                e2e_defer_reason,
6663                fix: None,
6664                blocking,
6665                answered,
6666                expected,
6667                clean,
6668                progressed: false,
6669                vote_split,
6670                reconsideration,
6671                verdict: None,
6672            };
6673            // The final vote per seat is its revote where reconsideration
6674            // ran and answered, its initial vote otherwise — the same
6675            // fallback `tally` uses for a judge whose private vote failed.
6676            round_record.verdict = ReviewVote::worst(
6677                round_record
6678                    .final_votes()
6679                    .into_iter()
6680                    .map(|(_, _, vote)| vote),
6681            );
6682            // Which commit and when magi actually attempted to check —
6683            // known the moment a command was dispatched against `head`,
6684            // whether or not it finished: a resource-blocked attempt still
6685            // targeted a specific commit at a specific time, and leaving
6686            // that unrecorded is exactly what made `verification_summary`
6687            // report a fresh attempt as "commit unknown ... recorded before
6688            // this was tracked", indistinguishable from a genuinely old,
6689            // untracked record. Only a deferred or unconfigured round never
6690            // ran at all and has nothing to record — see
6691            // `ReviewRound::verified_head`'s own doc.
6692            if !matches!(
6693                round_record.e2e_status(),
6694                E2eStatus::Deferred | E2eStatus::NotConfigured
6695            ) {
6696                round_record.verified_head = Some(head.clone());
6697                round_record.verified_at = Some(Timestamp::now());
6698            }
6699
6700            if incomplete {
6701                let missing: Vec<String> = round_record
6702                    .reviews
6703                    .iter()
6704                    .filter(|r| r.failed.is_some())
6705                    .map(|r| format!("review-{}", r.reviewer))
6706                    .collect();
6707                self.state.event(
6708                    "review",
6709                    format!(
6710                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6711                        missing.join(", ")
6712                    ),
6713                );
6714            }
6715
6716            if clean {
6717                self.state.event(
6718                    "review",
6719                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6720                        format!(
6721                            "round {round}: clean (warn policy, incomplete panel) — no \
6722                             blocking findings from the seats that answered, verification green"
6723                        )
6724                    } else if incomplete {
6725                        format!(
6726                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6727                             quorum) — no blocking findings from the seats that answered, \
6728                             verification green",
6729                            expected - answered
6730                        )
6731                    } else {
6732                        format!("round {round}: clean — no blocking findings, verification green")
6733                    },
6734                );
6735                self.state.reviews.push(round_record);
6736                self.state.status = RunStatus::Gating;
6737                self.state.save()?;
6738                return Ok(());
6739            }
6740
6741            // Nothing was raised and verification passed, but not every seat
6742            // answered and `round_is_clean` still refused to call it clean —
6743            // either a seat is missing for a reason other than its own quota
6744            // (a crash, a timeout, unparsable output — worth another try), or
6745            // every seat that could have answered lost its quota and nobody
6746            // is left to decide on: re-review rather than send the fixer
6747            // after a round with nothing to fix.
6748            if incomplete && blocking == 0 && e2e_ok {
6749                self.state.reviews.push(round_record);
6750                self.state.save()?;
6751                if round == max_rounds {
6752                    self.state.status = RunStatus::Blocked;
6753                    self.state.event(
6754                        "review",
6755                        format!(
6756                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6757                             refusing to call it clean",
6758                            expected - answered
6759                        ),
6760                    );
6761                    return Ok(());
6762                }
6763                continue;
6764            }
6765
6766            // Nothing for the fixer to act on (`blocking == 0`) and the only
6767            // reason this round is not clean is that magi itself never got
6768            // a command to run — the shared build cache, not the patch (see
6769            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6770            // the fixer would invite a change to appease contention that has
6771            // nothing to do with the diff, and would leave this attempt
6772            // sitting in the next round's prompt as if it were about an
6773            // earlier, superseded commit rather than what it actually is:
6774            // the same head, still waiting to be checked. Wait for it the
6775            // same way the final round's own contention is already handled,
6776            // whatever round this happens to be.
6777            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6778                self.state.reviews.push(round_record);
6779                return self
6780                    .stop_unresolved_review(
6781                        "the round's own verification could not run",
6782                        &shell,
6783                        &winner.worktree,
6784                    )
6785                    .await;
6786            }
6787
6788            if round == max_rounds {
6789                self.state.reviews.push(round_record);
6790                return self
6791                    .stop_reviewing(
6792                        &format!(
6793                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6794                        ),
6795                        &shell,
6796                        &winner.worktree,
6797                    )
6798                    .await;
6799            }
6800
6801            if !self.fix_review_round(&winner, round_record).await? {
6802                return Ok(());
6803            }
6804        }
6805        Ok(())
6806    }
6807
6808    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6809    /// False means admission or continuation policy stopped the review loop.
6810    async fn fix_review_round(
6811        &mut self,
6812        winner: &Candidate,
6813        mut round_record: ReviewRound,
6814    ) -> Result<bool> {
6815        let round = round_record.round;
6816        let shell = self.state.config.shell();
6817        let step = self.run_fix_round(winner, &mut round_record).await?;
6818        if matches!(step, FixStep::Resource) {
6819            self.stop_for_resource()?;
6820            return Ok(false);
6821        }
6822        if self.state.reviews.len() < round {
6823            self.state.reviews.push(round_record);
6824        } else {
6825            self.state.reviews[round - 1] = round_record;
6826        }
6827        self.state.save()?;
6828
6829        // The fixer's own report never came back this round, even after
6830        // `continue_fix_report`'s own budget was spent on it — not an
6831        // ordinary "no report" (dropped stream, quota, plain failure),
6832        // which already reads that way and is left to the existing round
6833        // budget. Stopping here, rather than opening another round, is
6834        // what keeps a next reviewer/fixer wave from ever being
6835        // dispatched onto `winner.worktree` while whatever the seat's
6836        // last call may still have running there is unaccounted for: no
6837        // process liveness check exists (and none is being added — see
6838        // AGENTS.md/this task's own scope), so the only way to honour
6839        // "nothing starts before a valid report returns" is to not start
6840        // anything further on this worktree from this run at all.
6841        if matches!(step, FixStep::Recorded { report_lost: true }) {
6842            self.stop_reviewing(
6843                "the fixer's adoption report never came back, even after resuming its \
6844                         own seat; refusing to start another round against the same worktree \
6845                         while that is unresolved",
6846                &shell,
6847                &winner.worktree,
6848            )
6849            .await?;
6850            return Ok(false);
6851        }
6852
6853        let streak = self
6854            .state
6855            .reviews
6856            .iter()
6857            .rev()
6858            .take_while(|r| !r.progressed)
6859            .count();
6860        if streak >= STAGNANT_LIMIT {
6861            self.stop_reviewing(
6862                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6863                &shell,
6864                &winner.worktree,
6865            )
6866            .await?;
6867            return Ok(false);
6868        }
6869        Ok(true)
6870    }
6871
6872    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6873    async fn run_fix_round(
6874        &mut self,
6875        winner: &Candidate,
6876        round_record: &mut ReviewRound,
6877    ) -> Result<FixStep> {
6878        let round = round_record.round;
6879        let max_rounds = self.state.config.graph.review_rounds;
6880        let run_id = self.state.id.clone();
6881        let prompts = self.state.config.prompts.clone();
6882        let language = self.state.config.graph.language.clone();
6883        let sessions = self.state.config.graph.sessions;
6884        let artifacts = self.merge_repair_artifacts();
6885        let base = self.landing_base();
6886        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6887        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6888        let this_round_verification = round_record.verification_summary(&head);
6889        let all_findings: Vec<_> = round_record
6890            .reviews
6891            .iter()
6892            .flat_map(|r| r.findings.iter().cloned())
6893            .collect();
6894        // Fix. The winner's own implementer seat continues its conversation:
6895        // the competition is over, so context is pure benefit now.
6896        let blocking_findings: Vec<_> = all_findings
6897            .iter()
6898            .filter(|f| f.severity.blocks())
6899            .cloned()
6900            .collect();
6901        let fix_prompt = prompt::fix_in(
6902            &self.state.instruction,
6903            &blocking_findings,
6904            this_round_verification.as_ref(),
6905            round,
6906            max_rounds,
6907            &language,
6908            &self.state.config.graph.github_language,
6909        );
6910        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6911        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6912        let (job, seat, out) = self
6913            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6914                prompt: fix_prompt.clone(),
6915                spec,
6916                seat,
6917                cwd: winner.worktree.clone(),
6918                timeout,
6919                allow_write: true,
6920                sessions,
6921                artifacts: artifacts.clone(),
6922                stem: format!("fix-{round}"),
6923                handover: None,
6924            })
6925            .await;
6926        // Not asked: the round is not recorded (so a resume repeats it),
6927        // no fixer was handed over to, and the seat is left as it was.
6928        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6929            return Ok(FixStep::Resource);
6930        }
6931        let agent_id = seat.agent.clone();
6932
6933        let mut fix = FixRecord {
6934            agent: agent_id,
6935            addressed: Vec::new(),
6936            rejected: Vec::new(),
6937            notes: String::new(),
6938            committed: false,
6939            failed: None,
6940            duration_ms: 0,
6941            continuation: None,
6942        };
6943        let mut continuation = ContinuationRecord::not_needed();
6944        let mut final_seat = seat.clone();
6945        match out {
6946            AgentOutcome::Ok(o) => {
6947                fix.duration_ms = o.duration_ms;
6948                let parsed = verdict::extract_json::<FixReport>(&o.text);
6949                // A parsed report standing next to a command this same
6950                // reply's own CLI never confirmed the exit status of is
6951                // not a resolved answer — the identical `CommandEvidence`
6952                // `state.jobs` renders, read here instead of only on
6953                // display, per the completion judgment and the shown
6954                // record needing to agree.
6955                let incomplete_reason = match &parsed {
6956                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6957                        "the reply parsed, but it reported a command whose own CLI never \
6958                             confirmed an exit status"
6959                            .to_owned(),
6960                    ),
6961                    Ok(_) => None,
6962                    Err(e) => Some(e.to_string()),
6963                };
6964                match incomplete_reason {
6965                    None => {
6966                        let report = parsed.expect("checked Ok above");
6967                        fix.addressed = report.addressed;
6968                        fix.rejected = report.rejected;
6969                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6970                    }
6971                    Some(reason) => {
6972                        let (resumed_seat, resolved, failure, cont) = self
6973                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6974                            .await;
6975                        fix.duration_ms += cont.cumulative_wait_ms;
6976                        continuation = cont;
6977                        final_seat = resumed_seat;
6978                        match resolved {
6979                            Some(report) => {
6980                                fix.addressed = report.addressed;
6981                                fix.rejected = report.rejected;
6982                                fix.notes =
6983                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6984                            }
6985                            None => fix.failed = failure,
6986                        }
6987                    }
6988                }
6989            }
6990            // The CLI's raw error JSON is not a fix report to parse.
6991            AgentOutcome::Dropped(o) => {
6992                fix.duration_ms = o.duration_ms;
6993                let why = o
6994                    .dropped
6995                    .as_ref()
6996                    .map(|d| d.why.as_str())
6997                    .unwrap_or("the CLI ended the stream without delivering its answer");
6998                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6999            }
7000            AgentOutcome::Quota(o) => {
7001                self.state.quota.push(QuotaLoss {
7002                    seat: final_seat.key.clone(),
7003                    node: "fix".to_owned(),
7004                    at: Timestamp::now(),
7005                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7006                });
7007                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7008            }
7009            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7010        }
7011        fix.continuation = Some(continuation);
7012        self.state.seats.insert(final_seat.key.clone(), final_seat);
7013        let coordination = self.coordinate(&winner.worktree).await?;
7014        if let Ok(r) = git::rescue_commit(
7015            &winner.worktree,
7016            &format!("magi: review round {round} fixes (uncommitted work)"),
7017        )
7018        .await
7019        {
7020            self.state.note_withheld("fix", &r.withheld);
7021        }
7022        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7023        fix.committed = after != before;
7024        // Judged by what `git` says moved against base, never by the
7025        // fixer's own `addressed`/`rejected` count — see
7026        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7027        // `patch` snapshot above: swallowing this error would default
7028        // `diff_after` to empty, which almost always differs from a
7029        // non-empty `patch` and reads as "progressed" — exactly backwards
7030        // for a `git` failure the stagnation check cannot see through.
7031        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7032        drop(coordination);
7033        let progressed = diff_after != patch;
7034        let commit_note = if fix.committed {
7035            "committed"
7036        } else {
7037            "NO new commit"
7038        };
7039        let tree_note = if progressed {
7040            "changed vs base"
7041        } else {
7042            "unchanged vs base"
7043        };
7044        self.state.event(
7045            "fix",
7046            match &fix.failed {
7047                // Distinct on purpose from "0 addressed, 0 rejected": the
7048                // fixer's own diff still landed (blocking counts do keep
7049                // falling round over round), only its adoption report did
7050                // not come back, so this must never read like every
7051                // finding was reviewed and declined.
7052                Some(reason) => {
7053                    format!(
7054                        "round {round}: fixer's adoption report was lost ({reason}); \
7055                             {commit_note}, tree {tree_note}"
7056                    )
7057                }
7058                None => format!(
7059                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7060                         {tree_note}{}",
7061                    fix.addressed.len(),
7062                    fix.rejected.len(),
7063                    if continuation.outcome == ContinuationOutcome::Resumed {
7064                        format!(
7065                            " (adoption report recovered after {} continuation(s))",
7066                            continuation.attempts
7067                        )
7068                    } else {
7069                        String::new()
7070                    },
7071                ),
7072            },
7073        );
7074        round_record.fix = Some(fix);
7075        round_record.progressed = progressed;
7076        Ok(FixStep::Recorded {
7077            report_lost: matches!(
7078                continuation.outcome,
7079                ContinuationOutcome::Exhausted
7080                    | ContinuationOutcome::QuotaLost
7081                    | ContinuationOutcome::NoSession
7082            ),
7083        })
7084    }
7085
7086    /// Decide, from the last recorded round's own verification, whether
7087    /// stopping the review loop is a hand-off or a genuine block.
7088    ///
7089    /// Called once the loop has given up trying — the round budget is spent,
7090    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7091    /// findings still open, never while a round is still clean or the
7092    /// incomplete-panel case handled inline above. Gate and e2e are facts
7093    /// about the tree; a lingering review finding is an opinion, and this
7094    /// workload's own `magi stats` puts reviewer precision low enough
7095    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7096    /// must not by itself stand between a green, verified change and the
7097    /// human who decides what to do with it. A red e2e is not an opinion, so
7098    /// that case still blocks, with the failing command and a tail of its
7099    /// output recorded here rather than left in `run.json` for someone to go
7100    /// find.
7101    ///
7102    /// A round that deferred its own e2e (see [`Config::graph`]'s
7103    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7104    /// only because nothing ran, and treating an empty list as a passing one
7105    /// here is exactly the "deferred painted green" bug this function exists
7106    /// to not have. When the last round's own verification never resolved —
7107    /// deferred on purpose, or a real attempt the shared build cache blocked
7108    /// — this makes (or retries) the real run, on the actual worktree this
7109    /// loop is about to stop touching, before deciding anything. A
7110    /// resource-blocked attempt is likewise never read as either green or
7111    /// red: it is evidence about the machine, not the patch (see
7112    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7113    /// blocked cache leaves this call without deciding rather than guessing
7114    /// — the caller retries on a later reentry.
7115    /// Record, once, that the review loop handed off over a blocking finding
7116    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7117    /// `land` asks the owner even with `land_approval` off. Called from every
7118    /// path that concludes `Gating`; a reentry keeps the first record.
7119    fn record_contested_handoff(&mut self) {
7120        if self.state.contested_handoff.is_some() {
7121            return;
7122        }
7123        let Some(contested) = self
7124            .state
7125            .reviews
7126            .last()
7127            .and_then(ReviewRound::contested_handoff)
7128        else {
7129            return;
7130        };
7131        self.state.event(
7132            "review",
7133            format!(
7134                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7135                 wait for the owner's approval",
7136                contested.findings.len(),
7137                contested.rejecters.len()
7138            ),
7139        );
7140        self.state.contested_handoff = Some(contested);
7141    }
7142
7143    async fn stop_unresolved_review(
7144        &mut self,
7145        why: &str,
7146        shell: &[String],
7147        worktree: &Path,
7148    ) -> Result<()> {
7149        self.stop_reviewing_inner(why, shell, worktree, true).await
7150    }
7151
7152    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7153        self.stop_reviewing_inner(why, shell, worktree, false).await
7154    }
7155
7156    async fn stop_reviewing_inner(
7157        &mut self,
7158        why: &str,
7159        shell: &[String],
7160        worktree: &Path,
7161        repairable: bool,
7162    ) -> Result<()> {
7163        let round_idx = self.state.reviews.len() - 1;
7164        // A deferred round and a resource-blocked one are the same shape
7165        // here: neither has a real result yet, and both get one more
7166        // attempt. Read off `e2e_status` — the single source for this —
7167        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
7168        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
7169        // still retries instead of being read as a settled result the
7170        // instant it stops being empty.
7171        let needs_catchup_run = matches!(
7172            self.state.reviews[round_idx].e2e_status(),
7173            E2eStatus::Deferred | E2eStatus::ResourceBlocked
7174        );
7175        if needs_catchup_run {
7176            let round = self.state.reviews[round_idx].round;
7177            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7178            let commands = self.state.config.verify.e2e.clone();
7179            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7180            let cache_dir = self.state.config.cache_dir();
7181            let context = format!(
7182                "round {round}: verification unresolved, catching up before the final decision"
7183            );
7184            let wait = self.repository_wait();
7185            let (outcomes, verify_retried) = with_cache_lease(
7186                &mut self.state,
7187                cache_dir.as_deref(),
7188                "e2e",
7189                "e2e",
7190                worktree,
7191                &attempted_head,
7192                timeout,
7193                &context,
7194                &wait,
7195                |state, budget| {
7196                    let shell = shell.to_vec();
7197                    let commands = commands.clone();
7198                    let context = context.clone();
7199                    async move {
7200                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7201                            .await
7202                    }
7203                },
7204            )
7205            .await?;
7206            let last = &mut self.state.reviews[round_idx];
7207            last.e2e = outcomes;
7208            last.verify_retried = verify_retried;
7209            // Always the commit and time this attempt actually targeted,
7210            // whether or not it happens to equal the reviewed `head` and
7211            // whether or not a command finished — see
7212            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7213            // attempt is recorded too, so a later reader sees "attempted
7214            // again at T2" rather than silence.
7215            last.verified_head = Some(attempted_head);
7216            last.verified_at = Some(Timestamp::now());
7217            if verify_inconclusive(&last.e2e) {
7218                // Still not a real result: `e2e_deferred` is left exactly
7219                // as it was, so `needs_catchup_run` above reads
7220                // `ResourceBlocked` (via `e2e_status`, which checks
7221                // `resource_blocked` before `e2e_deferred`) and retries
7222                // again on the next reentry, rather than recording
7223                // contention as a red e2e and blocking the run on it.
7224                self.state.save()?;
7225                return Ok(());
7226            }
7227            last.e2e_deferred = false;
7228        }
7229        let last = &self.state.reviews[round_idx];
7230        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7231
7232        match last.e2e_status() {
7233            E2eStatus::Failed => {
7234                let red: Vec<String> = last
7235                    .e2e
7236                    .iter()
7237                    .filter(|o| !o.ok())
7238                    .map(|o| {
7239                        format!(
7240                            "`{}` -> {:?}\n{}",
7241                            o.command,
7242                            o.code,
7243                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7244                        )
7245                    })
7246                    .collect();
7247                let can_repair = last.blocking == 0 && last.fix.is_none();
7248                self.state.reviews[round_idx].clean = false;
7249                self.state
7250                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7251                self.state.status = RunStatus::Blocked;
7252                if repairable
7253                    && can_repair
7254                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7255                {
7256                    self.state.status = RunStatus::Reviewing;
7257                }
7258            }
7259            // `needs_catchup_run` above already retried once this call; if
7260            // it is still blocked, this is magi's own admission it could
7261            // not get a command to run, never a verdict on the patch — the
7262            // run is left exactly where a later reentry can retry again.
7263            E2eStatus::ResourceBlocked => {
7264                self.state.event(
7265                    "review",
7266                    format!(
7267                        "{why}; e2e could not run (resource unavailable); not \
7268                         deciding yet"
7269                    ),
7270                );
7271            }
7272            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7273                self.state.event(
7274                    "review",
7275                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7276                );
7277                self.record_contested_handoff();
7278                self.state.status = RunStatus::Gating;
7279            }
7280        }
7281        self.state.save()?;
7282        Ok(())
7283    }
7284
7285    // ----------------------------------------------------------------- gate
7286
7287    async fn gate(&mut self) -> Result<()> {
7288        if self.hold_base_sync().await? {
7289            return Ok(());
7290        }
7291        // Judged by the review record itself, not by `status`: a solo
7292        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7293        // reentry (see `judge`), and trusting it here is exactly how a run
7294        // that exhausted its review budget got gated and merged a second
7295        // time around. `review_conclusion` recomputes the review loop's own
7296        // verdict from the round records themselves — `Gating` for a clean
7297        // round or a hand-off (see `stop_reviewing`), anything else means the
7298        // loop is still going or genuinely blocked.
7299        // A base the winner could not be replayed onto is a decision, not a
7300        // round: there is no landing tree to gate. Read as its own record for
7301        // the same reason the review verdict is.
7302        if self.state.status == RunStatus::Failed
7303            || self
7304                .state
7305                .base_sync
7306                .as_ref()
7307                .is_some_and(|s| s.conflict.is_some())
7308            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7309                != Some(RunStatus::Gating)
7310        {
7311            return Ok(());
7312        }
7313        if self.state.gate_ran {
7314            // `review_loop` derives its conclusion from the clean review
7315            // record on every reentry and therefore puts a completed run back
7316            // in `Gating`. A recorded gate is a stronger, terminal fact:
7317            // retain its original command output (or lack of any, for a repo
7318            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7319            // doc) and restore `Blocked` on a real failure rather than
7320            // pretending the command is still running or running it a second
7321            // time. `gate_ran == false` remains the only shape — unattempted,
7322            // or a resource-blocked retry — that may still need to execute a
7323            // command.
7324            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7325                self.state.status = RunStatus::Blocked;
7326                self.state.save()?;
7327            }
7328            return Ok(());
7329        }
7330        let Some(winner) = self.state.winner().cloned() else {
7331            return Ok(());
7332        };
7333        self.state.status = RunStatus::Gating;
7334        let mut outcomes = self.run_gate(&winner).await?;
7335        loop {
7336            // A resource-blocked outcome means the gate command never actually
7337            // ran - the shared build cache could not be acquired or confirmed
7338            // fresh in time - which is evidence about the machine, not about
7339            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7340            // Recording it as a red gate would mark a run `Blocked` on nothing
7341            // but contention magi has already logged; leaving `self.state.gate`
7342            // empty and `self.state.gate_ran` false instead keeps the shape
7343            // this function already treats as "still needs to run" (see the
7344            // early-return above), so the next call retries the command
7345            // rather than concluding anything.
7346            if verify_inconclusive(&outcomes) {
7347                self.state.save()?;
7348                return Ok(());
7349            }
7350            if outcomes.iter().all(CommandOutcome::ok) {
7351                break;
7352            }
7353            match self.gate_fix_round(&winner, &outcomes).await? {
7354                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7355                GateFix::Stop => break,
7356                GateFix::Defer => {
7357                    self.state.save()?;
7358                    return Ok(());
7359                }
7360            }
7361        }
7362        let passed = outcomes.iter().all(CommandOutcome::ok);
7363        self.state.gate = outcomes;
7364        self.state.gate_ran = true;
7365        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7366        if !passed {
7367            self.state.status = RunStatus::Blocked;
7368            let spent = self.state.gate_fixes.len();
7369            self.state.event(
7370                "gate",
7371                if spent == 0 {
7372                    "gate failed; not merging".to_owned()
7373                } else {
7374                    format!("gate failed after {spent} gate-fix round(s); not merging")
7375                },
7376            );
7377        }
7378        self.state.save()?;
7379        Ok(())
7380    }
7381
7382    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7383    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7384    /// after review is clean and never on a candidate awaiting judging.
7385    ///
7386    /// Never fails the run: a non-zero exit or timeout is a warning and a
7387    /// recorded outcome, and the gate remains the single arbiter. Nothing
7388    /// configured means nothing happens - no event, no commit. `commit_all`
7389    /// commits any leftover change under the neutral identity and returns
7390    /// `false` when the tree is clean, so no empty commit is ever made.
7391    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7392        let commands = state.config.verify.pre_gate.clone();
7393        if commands.is_empty() {
7394            return;
7395        }
7396        let shell = state.config.shell();
7397        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7398        let (outcomes, _) = run_commands(
7399            state,
7400            "pre_gate",
7401            "pre_gate",
7402            0,
7403            &shell,
7404            &commands,
7405            &winner.worktree,
7406            timeout,
7407        )
7408        .await;
7409        for o in &outcomes {
7410            if !o.ok() {
7411                tracing::warn!(
7412                    "pre_gate `{}` failed ({:?}); the gate decides",
7413                    o.command,
7414                    o.code
7415                );
7416            }
7417            state.event(
7418                "pre_gate",
7419                format!(
7420                    "`{}` -> {}",
7421                    o.command,
7422                    if o.ok() {
7423                        "pass".to_owned()
7424                    } else {
7425                        format!(
7426                            "FAIL ({:?})\n{}",
7427                            o.code,
7428                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7429                        )
7430                    }
7431                ),
7432            );
7433        }
7434        state.pre_gate = outcomes;
7435        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7436            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7437                Ok(head) => {
7438                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7439                    state.pre_gate_commit = Some(head);
7440                }
7441                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7442            },
7443            Ok(false) => {}
7444            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7445        }
7446        if let Err(e) = state.save() {
7447            tracing::warn!("could not persist the pre_gate record: {e:#}");
7448        }
7449    }
7450
7451    /// Run `verify.gate` once against the winner's current tree, logging one
7452    /// event per command. Empty when nothing is configured.
7453    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7454        let wait = self.repository_wait();
7455        run_landing_gate(&mut self.state, winner, &wait).await
7456    }
7457
7458    /// One bounded fix round for a failing gate.
7459    ///
7460    /// The fixer is told the failure came from the gate itself, not from a
7461    /// reviewer, and is shown the failed commands, their exit codes and a tail
7462    /// of their output - whatever `[verify].gate` holds, nothing here knows
7463    /// what those commands run. Only a normal non-zero exit that printed
7464    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7465    /// command or a full disk says nothing about the code, and a fixer sent
7466    /// after it can only appease the machine. The round is judged by what git
7467    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7468    /// again before the gate does, so a fix cannot trade a green gate for a
7469    /// red e2e unnoticed.
7470    async fn gate_fix_round(
7471        &mut self,
7472        winner: &Candidate,
7473        outcomes: &[CommandOutcome],
7474    ) -> Result<GateFix> {
7475        let cap = self.state.config.graph.gate_fix_rounds;
7476        let spent = self.state.gate_fixes.len();
7477        if spent >= cap {
7478            if cap > 0 {
7479                self.state.event(
7480                    "gate",
7481                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7482                );
7483            }
7484            return Ok(GateFix::Stop);
7485        }
7486        if !gate_fixable(outcomes) {
7487            self.state.event(
7488                "gate",
7489                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7490                 command or similar); not spending a fix round on it",
7491            );
7492            return Ok(GateFix::Stop);
7493        }
7494        let min_free = self.state.config.disk.min_free_bytes;
7495        if min_free > 0 {
7496            match crate::disk::free_bytes(&winner.worktree) {
7497                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7498                Ok(free) => {
7499                    self.state.event(
7500                        "gate",
7501                        format!(
7502                            "only {free} bytes free ({min_free} required by `[disk] \
7503                             min_free_bytes`); not spending a fix round on a failure the disk \
7504                             may explain"
7505                        ),
7506                    );
7507                    return Ok(GateFix::Stop);
7508                }
7509                Err(e) => {
7510                    self.state.event(
7511                        "gate",
7512                        format!("free disk space could not be measured ({e:#}); no fix round"),
7513                    );
7514                    return Ok(GateFix::Stop);
7515                }
7516            }
7517        }
7518
7519        let attempt = spent + 1;
7520        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7521        let base = self.landing_base();
7522        let fix_prompt = prompt::gate_fix_in(
7523            &self.state.instruction,
7524            &failed,
7525            attempt,
7526            cap,
7527            &self.state.config.graph.language,
7528            &self.state.config.graph.github_language,
7529        );
7530        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7531        let sessions = self.state.config.graph.sessions;
7532        let artifacts = self.merge_repair_artifacts();
7533        self.state.event(
7534            "gate",
7535            format!("gate failed; gate-fix round {attempt} of {cap}"),
7536        );
7537        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7538        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7539        // Persist before invoking the fixer: it may commit and then be
7540        // interrupted before its result (or the following e2e) is saved.
7541        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7542            head: before.clone(),
7543            passed: false,
7544            outcomes: Vec::new(),
7545        });
7546        self.state.save()?;
7547        let (_, seat, out) = self
7548            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7549                prompt: fix_prompt.clone(),
7550                spec,
7551                seat,
7552                cwd: winner.worktree.clone(),
7553                timeout,
7554                allow_write: true,
7555                sessions,
7556                artifacts: artifacts.clone(),
7557                stem: format!("gate-fix-{attempt}"),
7558                handover: None,
7559            })
7560            .await;
7561        // The fixer was never asked: no round is spent (nothing is recorded)
7562        // and a later reentry asks the same agent again.
7563        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7564            return Ok(GateFix::Defer);
7565        }
7566        let mut record = GateFixRecord {
7567            agent: seat.agent.clone(),
7568            failed,
7569            notes: String::new(),
7570            committed: false,
7571            error: None,
7572        };
7573        match out {
7574            AgentOutcome::Ok(o) => {
7575                // A missing report is not a failed fix: the round is judged
7576                // by the tree below, and the report only carries prose.
7577                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7578                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7579                }
7580            }
7581            AgentOutcome::Dropped(_) => {
7582                record.error = Some("the CLI dropped the stream".to_owned());
7583            }
7584            AgentOutcome::Quota(o) => {
7585                self.state.quota.push(QuotaLoss {
7586                    seat: seat.key.clone(),
7587                    node: "gate-fix".to_owned(),
7588                    at: Timestamp::now(),
7589                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7590                });
7591                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7592            }
7593            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7594        }
7595        self.state.seats.insert(seat.key.clone(), seat);
7596        let coordination = self.coordinate(&winner.worktree).await?;
7597        if let Ok(r) = git::rescue_commit(
7598            &winner.worktree,
7599            &format!("magi: gate fix {attempt} (uncommitted work)"),
7600        )
7601        .await
7602        {
7603            self.state.note_withheld("gate-fix", &r.withheld);
7604        }
7605        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7606        record.committed = after != before;
7607        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7608        let note = record.error.clone();
7609        self.state.gate_fixes.push(record);
7610        self.state.save()?;
7611        if !changed {
7612            self.state.event(
7613                "gate",
7614                match note {
7615                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7616                    None => format!("gate-fix round {attempt}: the tree did not change"),
7617                },
7618            );
7619            return Ok(GateFix::Stop);
7620        }
7621        self.state.event(
7622            "gate",
7623            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7624        );
7625
7626        drop(coordination);
7627        let wait = self.repository_wait();
7628        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7629    }
7630
7631    // ---------------------------------------------------------------- merge
7632
7633    /// One read-only rewrite by the summary's author; a text the gate still
7634    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7635    /// applies on their word or on silence. `None` means the run is parked on
7636    /// that question and nothing may be pushed or opened yet.
7637    async fn guarded_pr_message(
7638        &mut self,
7639        winner: &Candidate,
7640        facts: Option<&BranchFacts>,
7641        posting: bool,
7642    ) -> Result<Option<PrMessage>> {
7643        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7644        let head = match final_head.as_deref() {
7645            Some(h) => crate::pr_outcome::FinalHead::At(h),
7646            None => crate::pr_outcome::FinalHead::Unknown,
7647        };
7648        // Only a pull request that is about to be posted pays for a summary.
7649        let summarized = if posting {
7650            Some(self.summarize_change(winner, final_head.as_deref()).await)
7651        } else {
7652            None
7653        };
7654        // A busy shared resource is not a permanent cause: publishing an
7655        // "unavailable" description would stick (a later head move never
7656        // edits the body). Stop before anything is pushed or opened; the
7657        // run is `Blocked` and a resume asks the summary again.
7658        if self.state.resource_block.is_some() {
7659            self.state.event(
7660                "pr-summary",
7661                "a shared resource was busy; nothing was published, resume to summarize again",
7662            );
7663            return Ok(None);
7664        }
7665        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7666            anyhow::bail!(
7667                "the final commit moved during public description generation; retry before publishing"
7668            );
7669        }
7670        let note = match (&summarized, &self.state.change_summary) {
7671            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7672            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7673            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7674            (None, _) => ChangeNote::NotAsked,
7675        };
7676        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7677        if !posting {
7678            let identity = crate::scrub::Identity::current();
7679            return Ok(Some(PrMessage {
7680                title: crate::scrub::scrub(&pr.title, &identity),
7681                body: crate::scrub::scrub(&pr.body, &identity),
7682            }));
7683        }
7684        // This very text was already rejected and asked about: no second
7685        // judge call, rewrite or question, just carry the record on.
7686        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7687        if let Some(known) = self
7688            .state
7689            .github_text
7690            .clone()
7691            .filter(|g| g.fingerprint == raw_fp)
7692        {
7693            return self.settle_withheld(winner, &pr, known).await;
7694        }
7695        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7696        let mut violations =
7697            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7698        let identity = crate::scrub::Identity::current();
7699        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7700            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7701            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7702        {
7703            violations.push(crate::github_text::Violation::SensitiveData);
7704        }
7705        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7706        // only a language violation starts one; sensitive data alone goes
7707        // straight to the owner, with the redacted text as the fallback.
7708        let has_language = violations
7709            .iter()
7710            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7711        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7712            self.state.event(
7713                "github-text",
7714                if has_language {
7715                    format!("description rejected: {violations:?}; requesting one rewrite")
7716                } else {
7717                    format!("description rejected: {violations:?}; asking the owner")
7718                },
7719            );
7720            let mut rewritten = false;
7721            let mut rejected_rewrite = None;
7722            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7723            {
7724                let output = text;
7725                if let Some(title) = summary_title(&output) {
7726                    // Judge the redacted candidate that will actually be posted.
7727                    // Sensitive data alone must not discard a useful rewrite.
7728                    let raw_body = summary_without_title(&output);
7729                    let clean_title = crate::scrub::scrub(&title, &identity);
7730                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7731                    let redacted = clean_title != title || body != raw_body;
7732                    let title = clean_title;
7733                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7734                    if !complete {
7735                        self.state.event(
7736                            "github-text",
7737                            "description rewrite omitted required findings",
7738                        );
7739                    }
7740                    if !body.trim().is_empty() {
7741                        body.push_str("\n\n");
7742                        body.push_str(&public_pr_evidence(&self.state, head));
7743                    }
7744                    let marker = format!("magi:run/{}", self.state.id);
7745                    if !body.trim().is_empty() && !body.contains(&marker) {
7746                        body.push_str(&format!("\n\n{marker}"));
7747                    }
7748                    let rewrite = PrMessage {
7749                        title: title.clone(),
7750                        body: body.clone(),
7751                    };
7752                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7753                    let mut failed = crate::github_text::check_for(
7754                        &self.state.config.graph,
7755                        &title,
7756                        &body,
7757                        redecision,
7758                    );
7759                    if (!crate::github_text::shareable(&title)
7760                        || !crate::github_text::shareable(&body))
7761                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7762                    {
7763                        failed.push(crate::github_text::Violation::SensitiveData);
7764                    }
7765                    if complete
7766                        && !title.trim().is_empty()
7767                        && !body.trim().is_empty()
7768                        && !failed.is_empty()
7769                    {
7770                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7771                            title: title.clone(),
7772                            body: body.clone(),
7773                            failed: failed
7774                                .iter()
7775                                .map(|v| crate::github_text::category(*v).to_owned())
7776                                .collect(),
7777                        });
7778                    }
7779                    if complete
7780                        && !title.trim().is_empty()
7781                        && !raw_body.trim().is_empty()
7782                        && failed.is_empty()
7783                    {
7784                        decision = redecision;
7785                        pr = PrMessage { title, body };
7786                        rewritten = true;
7787                        self.state
7788                            .event("github-text", "description rewrite passed");
7789                        if redacted {
7790                            self.state.event(
7791                                "github-text",
7792                                "sensitive data removed from description rewrite",
7793                            );
7794                        }
7795                    }
7796                }
7797            }
7798            if !rewritten {
7799                self.state.event(
7800                    "github-text",
7801                    "description rewrite unavailable or rejected; asking the owner",
7802                );
7803                if let Some(w) =
7804                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7805                {
7806                    // A different text than the one asked about before: that
7807                    // question no longer describes anything.
7808                    self.retire_github_text_question("the withheld text changed");
7809                    let record = crate::run::GithubTextAsk {
7810                        rewrite: rejected_rewrite,
7811                        fingerprint: raw_fp,
7812                        title: w.title,
7813                        body: w.body,
7814                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7815                        question: None,
7816                        asks: 0,
7817                        resolved: false,
7818                        chosen_title: None,
7819                        choice: None,
7820                    };
7821                    return self.settle_withheld(winner, &pr, record).await;
7822                }
7823            }
7824        }
7825        let (title, body) =
7826            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7827        Ok(Some(PrMessage { title, body }))
7828    }
7829
7830    /// Describe the whole change from its final diff in either run mode.
7831    ///
7832    /// The summary is tied to the final commit and to the merge base of the
7833    /// resolved remote base; a record for any other pair is dropped and the
7834    /// agent asked again, so a description never outlives the diff it was
7835    /// written from. The seat is read-only and the prompt tells it to read the
7836    /// diff and the code around it. `[roles] synthesizer` selects the agents
7837    /// (unset: the default pick), each asked at most once, advancing on an
7838    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7839    /// carries the reason it is unavailable; nothing is invented then.
7840    async fn summarize_change(
7841        &mut self,
7842        winner: &Candidate,
7843        head: Option<&str>,
7844    ) -> std::result::Result<(), String> {
7845        let Some(head) = head else {
7846            return Err("the final commit could not be read".to_owned());
7847        };
7848        let remote = self.state.config.merge.remote.clone();
7849        let tracking = format!("{remote}/{}", self.state.base_branch);
7850        if !git::rev_exists(&winner.worktree, &tracking).await {
7851            return Err(format!("the remote base `{tracking}` could not be read"));
7852        }
7853        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7854            Ok(b) if !b.is_empty() => b,
7855            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7856        };
7857        let evidence = crate::pr_outcome::summary_evidence(
7858            &self.state,
7859            crate::pr_outcome::FinalHead::At(head),
7860        );
7861        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7862        if self
7863            .state
7864            .change_summary
7865            .as_ref()
7866            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7867        {
7868            return Ok(());
7869        }
7870        if self.state.change_summary.take().is_some() {
7871            self.state.event(
7872                "pr-summary",
7873                "the final diff or evidence changed; the earlier change summary was discarded",
7874            );
7875        }
7876        // Save the source snapshot even if no generation agent is available.
7877        if let Err(e) = write_artifact(
7878            &self.state,
7879            &format!("public-description-{fingerprint}.md"),
7880            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7881        ) {
7882            tracing::warn!("could not save public description evidence: {e:#}");
7883        }
7884        let chain = match agent::pick_chain(
7885            &self.state.config.agents,
7886            self.state.config.roles.synthesizer.as_ref(),
7887            &agent::installed,
7888            "synthesizer",
7889        ) {
7890            Ok(chain) => chain,
7891            Err(e) => {
7892                tracing::warn!("no agent for the change summary: {e:#}");
7893                self.state.event("pr-summary", "no summary agent available");
7894                return Err("no agent was available to summarize it".to_owned());
7895            }
7896        };
7897        let subjects = git::subjects(&winner.worktree, &base, head)
7898            .await
7899            .unwrap_or_default();
7900        let stat = git::diff_stat(&winner.worktree, &base, head)
7901            .await
7902            .unwrap_or_default();
7903        let stat: String = stat.chars().take(6_000).collect();
7904        let mut prompt = crate::prompt::change_summary_in(
7905            &base,
7906            head,
7907            &subjects,
7908            stat.trim(),
7909            &self.state.config.graph.github_language,
7910        );
7911        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7912        let sessions = self.state.config.graph.sessions;
7913        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7914        let prompts = self.state.config.prompts.clone();
7915        let cache = self.state.config.cache_dir();
7916        let run = self.state.id.clone();
7917        for (i, spec) in chain.into_iter().enumerate() {
7918            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7919            let github_spec = match agent::for_github_text(
7920                &spec,
7921                &self.state.config.graph.github_language,
7922                &winner.worktree,
7923            ) {
7924                Ok(spec) => spec,
7925                Err(e) => {
7926                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7927                    continue;
7928                }
7929            };
7930            let job = SeatJob {
7931                spec: github_spec,
7932                seat,
7933                cwd: winner.worktree.clone(),
7934                prompt: prompt.clone(),
7935                timeout,
7936                allow_write: false,
7937                sessions,
7938                artifacts: agent::artifacts_dir(&self.state.dir()),
7939                stem: if i == 0 {
7940                    "change-summary".to_owned()
7941                } else {
7942                    format!("change-summary-{}", spec.id)
7943                },
7944                handover: None,
7945            };
7946            let ctx = WaveCtx {
7947                run: &run,
7948                node: "pr-summary",
7949                prompts: &prompts,
7950                cache: cache.as_deref(),
7951                round: None,
7952                carry_seats: false,
7953                pauses: Some((&self.pause, &self.interrupt)),
7954            };
7955            let (seat, outcome) =
7956                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7957            self.state.seats.insert(seat.key.clone(), seat);
7958            let text = match outcome {
7959                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7960                AgentOutcome::ResourceBlocked(why) => {
7961                    // No agent was asked: the run stops resumably before
7962                    // anything is published (see `guarded_pr_message`).
7963                    if self.state.resource_block.is_none() {
7964                        self.state.resource_block = Some(crate::run::ResourceBlock {
7965                            cause: crate::run::ResourceBlockCause::Other,
7966                            node: "pr-summary".to_owned(),
7967                            kind: "shared resource".to_owned(),
7968                            detail: why,
7969                            at: Timestamp::now(),
7970                        });
7971                    }
7972                    return Err("a shared resource was busy".to_owned());
7973                }
7974                _ => continue,
7975            };
7976            let blind = &self.state.config.blind;
7977            let Some(title) = summary_title(&text) else {
7978                continue;
7979            };
7980            let body: String = summary_without_title(&text)
7981                .lines()
7982                .filter(|l| !l.contains("magi:run/"))
7983                .collect::<Vec<_>>()
7984                .join("\n");
7985            let body = blind::sanitize_prose(body.trim(), blind);
7986            let title = blind::sanitize_prose(&title, blind);
7987            if body.trim().is_empty() || title.trim().is_empty() {
7988                continue;
7989            }
7990            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7991                continue;
7992            }
7993            // The branch must still be the one the agent was shown.
7994            if git::rev_parse(&winner.worktree, "HEAD")
7995                .await
7996                .ok()
7997                .as_deref()
7998                != Some(head)
7999            {
8000                return Err("the branch moved while the summary was written".to_owned());
8001            }
8002            if git::merge_base(&winner.worktree, &tracking, head)
8003                .await
8004                .ok()
8005                .as_deref()
8006                != Some(base.as_str())
8007                || crate::pr_outcome::summary_evidence(
8008                    &self.state,
8009                    crate::pr_outcome::FinalHead::At(head),
8010                ) != evidence
8011            {
8012                return Err("the summary evidence changed while it was written".to_owned());
8013            }
8014            self.state.change_summary = Some(crate::run::ChangeSummary {
8015                head: head.to_owned(),
8016                base,
8017                evidence: fingerprint,
8018                title,
8019                body,
8020            });
8021            self.state.event(
8022                "pr-summary",
8023                format!("change summary written by {} for the final diff", spec.id),
8024            );
8025            if let Err(e) = self.state.save() {
8026                tracing::warn!("could not save the change summary: {e:#}");
8027            }
8028            return Ok(());
8029        }
8030        self.state
8031            .event("pr-summary", "no agent produced a change summary");
8032        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8033    }
8034
8035    /// One read-only rewrite request for a withheld description. The
8036    /// implementer's own conversation is tried first when it exists and can be
8037    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8038    /// writes it in a separate conversation, skipping the implementer's agent.
8039    /// The prompt carries the whole text, so no agent needs any memory. Each
8040    /// agent is asked at most once and the chain advances on an error, quota,
8041    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8042    /// that has run out of quota must not end the rewrite. Nothing is recorded
8043    /// as a handover.
8044    async fn rewrite_pr_text(
8045        &mut self,
8046        winner: &Candidate,
8047        pr: &PrMessage,
8048        violations: &[crate::github_text::Violation],
8049    ) -> Option<String> {
8050        let prompt = format!(
8051            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8052            pr.title, pr.body
8053        );
8054        let sessions = self.state.config.graph.sessions;
8055        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8056        // (spec, the resumed implementer's seat). A fallback seat is created
8057        // only when its turn comes, so an unused fallback leaves no record.
8058        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8059        let key = format!("impl-{}", winner.label);
8060        if let Some(spec) = self
8061            .state
8062            .config
8063            .agents
8064            .iter()
8065            .find(|a| a.id == winner.agent)
8066            .cloned()
8067            && let Some(existing) = self.state.seats.get(&key)
8068            && existing.agent == spec.id
8069            && agent::has_session(spec.kind, existing, sessions)
8070        {
8071            attempts.push((spec, Some(existing.clone())));
8072        }
8073        match agent::pick_chain(
8074            &self.state.config.agents,
8075            self.state.config.roles.language_judge.as_ref(),
8076            &agent::installed,
8077            "language_judge",
8078        ) {
8079            Ok(chain) => {
8080                for spec in chain {
8081                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8082                        continue;
8083                    }
8084                    attempts.push((spec, None));
8085                }
8086            }
8087            Err(e) if attempts.is_empty() => {
8088                tracing::warn!("no agent for the description rewrite: {e:#}");
8089                self.state
8090                    .event("github-text", "no rewrite agent available");
8091                return None;
8092            }
8093            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8094        }
8095        let prompts = self.state.config.prompts.clone();
8096        let cache = self.state.config.cache_dir();
8097        let run = self.state.id.clone();
8098        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8099            let resumed = resumed_seat.is_some();
8100            let seat = resumed_seat.unwrap_or_else(|| {
8101                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8102            });
8103            let github_spec = match agent::for_github_text(
8104                &spec,
8105                &self.state.config.graph.github_language,
8106                &winner.worktree,
8107            ) {
8108                Ok(spec) => spec,
8109                Err(e) => {
8110                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8111                    continue;
8112                }
8113            };
8114            let job = SeatJob {
8115                spec: github_spec,
8116                seat,
8117                cwd: winner.worktree.clone(),
8118                prompt: prompt.clone(),
8119                timeout: retry_budget(timeout_full, resumed),
8120                allow_write: false,
8121                sessions,
8122                artifacts: agent::artifacts_dir(&self.state.dir()),
8123                stem: if i == 0 {
8124                    "github-text-rewrite".to_owned()
8125                } else {
8126                    format!("github-text-rewrite-{}", spec.id)
8127                },
8128                handover: None,
8129            };
8130            let ctx = WaveCtx {
8131                run: &run,
8132                node: "github-text",
8133                prompts: &prompts,
8134                cache: cache.as_deref(),
8135                round: None,
8136                carry_seats: false,
8137                pauses: Some((&self.pause, &self.interrupt)),
8138            };
8139            let (seat, outcome) =
8140                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8141            self.state.seats.insert(seat.key.clone(), seat);
8142            match outcome {
8143                AgentOutcome::Ok(o)
8144                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8145                {
8146                    return Some(o.text);
8147                }
8148                AgentOutcome::ResourceBlocked(_) => return None,
8149                _ => {}
8150            }
8151        }
8152        None
8153    }
8154
8155    /// Abandon the standing posting-gate question, if any.
8156    fn retire_github_text_question(&mut self, why: &str) {
8157        let Some(id) = self
8158            .state
8159            .github_text
8160            .as_ref()
8161            .and_then(|g| g.question.clone())
8162        else {
8163            return;
8164        };
8165        let _ = ask::Questions::open().update(&id, |q| {
8166            q.abandon(why);
8167            Ok(())
8168        });
8169    }
8170
8171    /// Carry a withheld text to its end: file the question, park while it is
8172    /// open, and on an answer, silence or timeout pick the text to post. The
8173    /// decision is saved before it is used, so it applies once.
8174    async fn settle_withheld(
8175        &mut self,
8176        winner: &Candidate,
8177        pr: &PrMessage,
8178        mut g: crate::run::GithubTextAsk,
8179    ) -> Result<Option<PrMessage>> {
8180        use crate::github_text as gt;
8181        let store = ask::Questions::open();
8182        let timeout = self.state.config.graph.answer_timeout;
8183        if !g.resolved {
8184            let mut standing = g
8185                .question
8186                .as_ref()
8187                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8188            if let Some(q) = standing.as_ref()
8189                && q.status.open()
8190                && gt::expired(q, timeout)
8191            {
8192                let why = format!("no answer within {}s of asking", timeout.max(1));
8193                standing = store
8194                    .update(&q.id, |q| {
8195                        q.abandon(&why);
8196                        Ok(())
8197                    })
8198                    .ok()
8199                    .map(|(q, ())| q);
8200            }
8201            let mut ask_again: Option<Option<Vec<String>>> = None;
8202            match standing {
8203                // Recorded but never written (a stop between the two saves),
8204                // or removed: ask again without spending the retry.
8205                None if g.asks <= 1 => {
8206                    g.asks = 0;
8207                    ask_again = Some(None);
8208                }
8209                None => {
8210                    g.resolved = true;
8211                    g.chosen_title = None;
8212                    g.choice = Some(GithubTextChoice::Fallback);
8213                }
8214                Some(q) if q.status.open() => {
8215                    self.park_on_github_text(&g)?;
8216                    return Ok(None);
8217                }
8218                Some(q) => match gt::read_reply(&q) {
8219                    gt::Reply::Fallback => {
8220                        g.resolved = true;
8221                        g.chosen_title = None;
8222                        g.choice = Some(GithubTextChoice::Fallback);
8223                    }
8224                    gt::Reply::Original => {
8225                        g.resolved = true;
8226                        g.chosen_title = None;
8227                        g.choice = Some(GithubTextChoice::Original);
8228                    }
8229                    gt::Reply::Rewrite => {
8230                        g.resolved = true;
8231                        g.chosen_title = None;
8232                        g.choice = Some(if g.rewrite.is_some() {
8233                            GithubTextChoice::Rewrite
8234                        } else {
8235                            GithubTextChoice::Fallback
8236                        });
8237                    }
8238                    gt::Reply::Title(text) => {
8239                        let first = text
8240                            .lines()
8241                            .map(str::trim)
8242                            .find(|l| !l.is_empty())
8243                            .unwrap_or("");
8244                        let decision = if first.is_empty() {
8245                            None
8246                        } else {
8247                            let candidate = PrMessage {
8248                                title: first.to_owned(),
8249                                body: String::new(),
8250                            };
8251                            self.judge_pr_language(&winner.worktree, &candidate).await
8252                        };
8253                        match gt::vet_title(
8254                            &text,
8255                            decision,
8256                            self.state.config.graph.github_english(),
8257                        ) {
8258                            Ok(title) => {
8259                                g.resolved = true;
8260                                g.chosen_title = Some(title);
8261                                g.choice = Some(GithubTextChoice::MyTitle);
8262                            }
8263                            Err(cats) if g.asks < 2 => {
8264                                ask_again = Some(Some(
8265                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8266                                ))
8267                            }
8268                            Err(_) => {
8269                                g.resolved = true;
8270                                g.chosen_title = None;
8271                                g.choice = Some(GithubTextChoice::Fallback);
8272                            }
8273                        }
8274                    }
8275                },
8276            }
8277            if let Some(rejected) = ask_again {
8278                // A retry (after a rejected replacement title) is still about
8279                // the original text: the same ORIGINAL / FALLBACK comparison.
8280                let w = gt::Withheld {
8281                    title: g.title,
8282                    body: g.body,
8283                    categories: g.categories.clone(),
8284                };
8285                // Named by the original text's fingerprint alone, so a resume
8286                // rewrites the same file and the detail stays identical.
8287                let artifact = match crate::run::write_artifact(
8288                    &self.state,
8289                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8290                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8291                ) {
8292                    Ok(p) => Some(p),
8293                    Err(e) => {
8294                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8295                        None
8296                    }
8297                };
8298                let (detail, comparison) = gt::question_content(
8299                    &w,
8300                    &pr.title,
8301                    &pr.body,
8302                    &self.state.id,
8303                    rejected.as_deref(),
8304                    artifact.as_deref(),
8305                    g.rewrite.as_ref(),
8306                );
8307                let mut q = ask::Question::new(
8308                    self.state.id.clone(),
8309                    gt::ASK_NODE.to_owned(),
8310                    gt::ASK_SEAT.to_owned(),
8311                    gt::question_summary(&self.state.config.graph.language, &w),
8312                    detail,
8313                    gt::question_choices(&w, g.rewrite.as_ref()),
8314                );
8315                q.posting_comparison = Some(comparison);
8316                q.answer_timeout = timeout;
8317                // Recorded before it is written: a stop in between leaves a
8318                // record naming a question the resume then files again.
8319                g.question = Some(q.id.clone());
8320                g.asks += 1;
8321                self.state.github_text = Some(g.clone());
8322                self.state.event(
8323                    "github-text",
8324                    format!("asking the owner about the withheld text ({})", q.short()),
8325                );
8326                self.state.save()?;
8327                store
8328                    .put_executable(&mut q)
8329                    .context("file the posting-gate question")?;
8330                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8331                    tracing::warn!(
8332                        "could not notify about posting-gate question {}: {e:#}",
8333                        q.short()
8334                    );
8335                }
8336                self.park_on_github_text(&g)?;
8337                return Ok(None);
8338            }
8339        }
8340        let resolved = g.chosen_title.clone();
8341        let original = g.choice == Some(GithubTextChoice::Original);
8342        // Field names only: the event log never records the text.
8343        self.state.event(
8344            "github-text",
8345            if original {
8346                format!(
8347                    "posting the original text for: {}",
8348                    match (g.title, g.body) {
8349                        (true, true) => "title, description",
8350                        (true, false) => "title",
8351                        _ => "description",
8352                    }
8353                )
8354            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8355                "posting the saved rewrite title and description".to_owned()
8356            } else if resolved.is_some() {
8357                "posting the owner's replacement title".to_owned()
8358            } else {
8359                "posting the neutral text for the withheld fields".to_owned()
8360            },
8361        );
8362        self.state.github_text = Some(g.clone());
8363        self.state.save()?;
8364        if g.choice == Some(GithubTextChoice::Rewrite)
8365            && let Some(r) = g.rewrite.as_ref()
8366        {
8367            return Ok(Some(PrMessage {
8368                title: r.title.clone(),
8369                body: r.body.clone(),
8370            }));
8371        }
8372        let w = gt::Withheld {
8373            title: g.title,
8374            body: g.body,
8375            categories: g.categories.clone(),
8376        };
8377        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8378        if let Some(t) = resolved {
8379            title = t;
8380        }
8381        // Every withheld field is now neutral or vetted English.
8382        let english = gt::LanguageDecision {
8383            title_english: true,
8384            body_english: true,
8385            body_complete: true,
8386        };
8387        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8388        if original {
8389            // The owner's explicit choice overrides the gate for the fields it
8390            // was made for; the others keep what the gate produced.
8391            if g.title {
8392                title = pr.title.clone();
8393            }
8394            if g.body {
8395                body = pr.body.clone();
8396            }
8397        }
8398        Ok(Some(PrMessage { title, body }))
8399    }
8400
8401    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8402        self.state.github_text = Some(g.clone());
8403        self.state.end_pass_parked(ParkCause::Question);
8404        self.state.parked = true;
8405        self.state.event(
8406            "github-text",
8407            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8408        );
8409        self.state.save()
8410    }
8411
8412    /// Ask `[roles] language_judge` about this text and record which source
8413    /// decided; the text itself is never recorded.
8414    async fn judge_pr_language(
8415        &mut self,
8416        cwd: &Path,
8417        pr: &PrMessage,
8418    ) -> Option<crate::github_text::LanguageDecision> {
8419        self.state.config.roles.language_judge.as_ref()?;
8420        if !self.state.config.graph.language_gate() {
8421            return None;
8422        }
8423        let decision =
8424            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8425        self.state.event(
8426            "github-text",
8427            if decision.is_some() {
8428                "language decided by the language judge"
8429            } else {
8430                "language judge unavailable; using built-in heuristics"
8431            },
8432        );
8433        decision
8434    }
8435
8436    async fn merge(&mut self) -> Result<()> {
8437        if self.hold_base_sync().await? {
8438            return Ok(());
8439        }
8440        // Same reasoning as `gate`: ask the review and gate records directly
8441        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8442        // skip can rewrite on reentry to something that no longer says
8443        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8444        // so a hand-off (open findings, green verification) reaches merge
8445        // exactly like a genuinely clean round does.
8446        //
8447        // A run resumed mid-`land` never reaches here at all: `execute`
8448        // recognises `RunStatus::Landing` before it even calls `prep`, and
8449        // routes straight to `run_land` instead. That has to happen a level
8450        // up from this function, not with a check in here, because
8451        // `review_loop`'s own status recomputation (see its doc) runs
8452        // *before* `merge` on every reentry and would otherwise overwrite
8453        // the `Landing` marker with `Gating` before this node ever saw it.
8454        if self
8455            .state
8456            .base_sync
8457            .as_ref()
8458            .is_some_and(|s| s.conflict.is_some())
8459            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8460                != Some(RunStatus::Gating)
8461            // `gate_ran == false` is not "passed" - `gate` leaves it false
8462            // both before it has ever run and when its last attempt was
8463            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8464            // permission to merge on nothing but the review record. Only a
8465            // gate that actually ran may proceed; an intentional local
8466            // review can also finish with verification explicitly not configured.
8467            // `RunState::gate_status` computes that distinction.
8468            || !self.state.gate_allows_progress()
8469        {
8470            return Ok(());
8471        }
8472        // This node's own record, not `status`: `status == Ready` is not
8473        // unique to the harmless `MergeMode::None` path this line was
8474        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8475        // run's PR was closed without merging — and on that run `mode` is
8476        // still `Pr`, so a reentry that fell through here would push and
8477        // open a second pull request. `self.state.merge` is set exactly once
8478        // this node (or `land`) has already produced a verdict, under every
8479        // mode, which is what "already done" actually means here.
8480        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8481            return Ok(());
8482        }
8483        let Some(winner) = self.state.winner().cloned() else {
8484            return Ok(());
8485        };
8486        let repo = self.state.repo.clone();
8487        let base = self.state.base_branch.clone();
8488        let mode = self.state.config.merge.mode;
8489        let style = self.state.config.merge.style;
8490        let facts = if is_review_run(&self.state) {
8491            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8492            let start = review_base(
8493                &repo,
8494                &self.state.config.merge.remote,
8495                &base,
8496                &self.state.base_commit,
8497                &winner.branch,
8498            )
8499            .await;
8500            branch_facts(&repo, &start, &winner.branch).await
8501        } else {
8502            None
8503        };
8504        // `None` when the base could not be freshly read: then an adopted
8505        // pull request's title is left alone.
8506        let leaked = if is_review_run(&self.state) {
8507            leaked_subjects(&self.state, &winner.branch).await
8508        } else {
8509            Some(Vec::new())
8510        };
8511        // `None`: parked on the owner's word about a withheld title or body.
8512        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8513        // the resume arrives here again.
8514        let Some(pr) = self
8515            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8516            .await?
8517        else {
8518            return Ok(());
8519        };
8520        let message = pr.commit_message();
8521
8522        let outcome = match mode {
8523            MergeMode::None => MergeOutcome {
8524                mode,
8525                ok: true,
8526                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8527                empty: false,
8528            },
8529            MergeMode::Pr | MergeMode::Local
8530                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8531            {
8532                MergeOutcome {
8533                    mode,
8534                    ok: false,
8535                    detail: empty_candidate_detail(&self.state, &base),
8536                    empty: true,
8537                }
8538            }
8539            MergeMode::Local => {
8540                let on = git::current_branch(&repo).await?;
8541                if on.as_deref() != Some(base.as_str()) {
8542                    MergeOutcome {
8543                        mode,
8544                        ok: false,
8545                        detail: format!(
8546                            "{} has {} checked out, not the base branch {base}",
8547                            repo.display(),
8548                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8549                        ),
8550                        empty: false,
8551                    }
8552                } else if !git::is_clean(&repo).await? {
8553                    MergeOutcome {
8554                        mode,
8555                        ok: false,
8556                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8557                        empty: false,
8558                    }
8559                } else {
8560                    let out = match style {
8561                        MergeStyle::Merge => {
8562                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8563                        }
8564                        MergeStyle::Squash => {
8565                            git::merge_squash(&repo, &winner.branch, &message).await?
8566                        }
8567                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8568                    };
8569                    MergeOutcome {
8570                        mode,
8571                        ok: out.ok(),
8572                        detail: if out.ok() { out.stdout } else { out.stderr },
8573                        empty: false,
8574                    }
8575                }
8576            }
8577            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8578                self.state.pr_publication_pending = true;
8579                self.state.save()?;
8580                let out =
8581                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8582                self.state.pr_publication_pending = out.is_err();
8583                MergeOutcome {
8584                    mode,
8585                    ok: out.is_ok(),
8586                    detail: match out {
8587                        Ok(url) => url,
8588                        Err(e) => format!("{e:#}"),
8589                    },
8590                    empty: false,
8591                }
8592            }
8593            MergeMode::Pr => {
8594                let remote = self.state.config.merge.remote.clone();
8595                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8596                if !pushed.ok() {
8597                    MergeOutcome {
8598                        mode,
8599                        ok: false,
8600                        detail: pushed.stderr,
8601                        empty: false,
8602                    }
8603                } else {
8604                    // A retry or resume of a run whose branch already has an
8605                    // open pull request adopts it rather than failing on a
8606                    // duplicate. Only this winner branch into this base:
8607                    // `branch_for` derives the name from the run id, so a
8608                    // different run's pull request never matches.
8609                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8610                    let out = match pr_merge_plan(found) {
8611                        PrPlan::Create => {
8612                            gh_pr_create(
8613                                &winner.worktree,
8614                                &base,
8615                                &winner.branch,
8616                                &pr.title,
8617                                &pr.body,
8618                            )
8619                            .await
8620                        }
8621                        PrPlan::Adopt { url, title } => {
8622                            self.state
8623                                .event("merge", format!("Pr: adopted open pull request {url}"));
8624                            if self.state.github_text.as_ref().is_some_and(|g| {
8625                                g.resolved
8626                                    && g.choice == Some(GithubTextChoice::Rewrite)
8627                                    && g.rewrite.is_some()
8628                            }) {
8629                                // The owner chose the entire pair, even if the
8630                                // existing title already matches or was edited.
8631                                land::set_pr_text_verbatim(
8632                                    &winner.worktree,
8633                                    &url,
8634                                    &pr.title,
8635                                    &pr.body,
8636                                )
8637                                .await
8638                                .map(|()| url)
8639                            } else {
8640                                if title != pr.title
8641                                    && (!is_review_run(&self.state)
8642                                        || leaked
8643                                            .as_deref()
8644                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8645                                    && let Err(e) =
8646                                        if self.state.github_text.as_ref().is_some_and(|g| {
8647                                            g.resolved
8648                                                && g.title
8649                                                && g.choice == Some(GithubTextChoice::Original)
8650                                        }) {
8651                                            // The owner chose to post this title unmodified.
8652                                            land::set_pr_title_verbatim(
8653                                                &winner.worktree,
8654                                                &url,
8655                                                &pr.title,
8656                                            )
8657                                            .await
8658                                        } else {
8659                                            land::set_pr_title(
8660                                                &mut self.state,
8661                                                &winner.worktree,
8662                                                &url,
8663                                                &pr.title,
8664                                            )
8665                                            .await
8666                                        }
8667                                {
8668                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8669                                    self.state
8670                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8671                                }
8672                                Ok(url)
8673                            }
8674                        }
8675                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8676                    };
8677                    match out {
8678                        Ok(url) => MergeOutcome {
8679                            mode,
8680                            ok: true,
8681                            detail: url,
8682                            empty: false,
8683                        },
8684                        Err(e) => MergeOutcome {
8685                            mode,
8686                            ok: false,
8687                            detail: e.to_string(),
8688                            empty: false,
8689                        },
8690                    }
8691                }
8692            }
8693        };
8694
8695        self.state.status = match (mode, outcome.ok) {
8696            (MergeMode::None, _) => RunStatus::Ready,
8697            (_, true) if outcome.empty => RunStatus::Ready,
8698            (MergeMode::Pr, true) => RunStatus::Ready,
8699            (_, true) => RunStatus::Merged,
8700            (_, false) => RunStatus::Blocked,
8701        };
8702        self.state.event(
8703            "merge",
8704            format!(
8705                "{:?}: {}",
8706                mode,
8707                outcome.detail.lines().next().unwrap_or("")
8708            ),
8709        );
8710        self.state.merge = Some(outcome);
8711        self.state.save()?;
8712
8713        // The PR is open and the run would historically stop here, leaving the
8714        // operator to watch checks, feed review comments back to a fixer, and
8715        // merge. That was done by hand six times in one session before this
8716        // existed. Opt-in, because merging is the one irreversible thing magi
8717        // can do to a repository.
8718        if self.state.config.graph.land
8719            && mode == MergeMode::Pr
8720            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8721        {
8722            self.run_land().await?;
8723        }
8724        // `run_land` may have left `status` at `Landing` - still waiting on
8725        // CI or the owner's approval, not actually settled - so this has to
8726        // read whatever `status` ended up as here, not the `Merged` this
8727        // function set a few lines up.
8728        self.settle_questions();
8729        Ok(())
8730    }
8731
8732    /// Enter `land`.
8733    ///
8734    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8735    /// a resumed run's re-entry. `land::land` itself is what serialises the
8736    /// two git-mutating moments inside the loop — the rebase push and
8737    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8738    /// to hold a lock across the whole call, and doing so would serialise
8739    /// this run's CI wait against a *different* run's land-approval resume
8740    /// in the same repository, which is exactly the "must not wait on
8741    /// another task" property the daemon's slot-freeing exists to give.
8742    async fn run_land(&mut self) -> Result<()> {
8743        crate::pr_head::ensure_owner(&self.state)?;
8744        let url = self
8745            .state
8746            .pr
8747            .as_ref()
8748            .map(|pr| pr.url.clone())
8749            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8750            .unwrap_or_default();
8751        let url = url.lines().next().unwrap_or("").trim().to_owned();
8752        if !url.starts_with("http") {
8753            return Ok(());
8754        }
8755        // A land failure is not a lost run: the work is on a branch and the
8756        // pull request is open, which is exactly where a human takes over.
8757        loop {
8758            match land::land(&mut self.state, &url).await {
8759                Ok(_)
8760                    if self
8761                        .state
8762                        .merge_repairs
8763                        .last()
8764                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8765                {
8766                    if self.repair_merge_review().await? {
8767                        continue;
8768                    }
8769                }
8770                Ok(pr) if self.state.parked => {
8771                    // `land` already saved the parked marker; nothing here
8772                    // overrides `status` back to a terminal value while an
8773                    // approval is still outstanding.
8774                    let _ = pr;
8775                }
8776                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8777                    self.state.status = RunStatus::Superseded;
8778                    self.state.save()?;
8779                }
8780                Ok(pr) => {
8781                    self.state.status = match pr.state {
8782                        land::PrLifecycle::Merged => RunStatus::Merged,
8783                        _ => RunStatus::Blocked,
8784                    };
8785                    // Downstream of a confirmed merge only - see
8786                    // `bump::should_release_bump`'s own doc for why this one
8787                    // check covers all three of `land`'s success paths.
8788                    // Best-effort: the run already landed, so a failure here
8789                    // (the decision call, `gh`, `cargo`) is recorded and never
8790                    // turns a landed run into a failed one.
8791                    if bump::should_release_bump(self.state.status)
8792                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8793                    {
8794                        // The event is the run's own record. Not-eligible cases
8795                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8796                        // `Err` is a bump that was tried and failed:
8797                        // `after_merge` itself raises the operator notice for
8798                        // that, whether or not a release PR exists yet.
8799                        self.state
8800                            .event("bump", format!("release bump skipped: {e:#}"));
8801                    }
8802                    // Independent of the bump, and best-effort in the same way:
8803                    // findings the merge left open become follow-up tasks.
8804                    if self.state.status == RunStatus::Merged {
8805                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8806                    }
8807                    self.state.save()?;
8808                }
8809                Err(e) => {
8810                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8811                        return Err(e);
8812                    }
8813                    self.state.status = RunStatus::Blocked;
8814                    self.state.event("land", format!("gave up: {e}"));
8815                    self.state.save()?;
8816                }
8817            }
8818            break;
8819        }
8820        Ok(())
8821    }
8822
8823    /// One repair cycle stays under the existing driver/task claim. An
8824    /// interrupted fixer is rolled back and held rather than run twice.
8825    async fn repair_merge_review(&mut self) -> Result<bool> {
8826        use crate::run::MergeRepairPhase as Phase;
8827        crate::pr_head::ensure_owner(&self.state)?;
8828        let i = self.state.merge_repairs.len() - 1;
8829        let request = self.state.merge_repairs[i].clone();
8830        if request.phase == Phase::Fixing {
8831            let restored = match self.state.winner().cloned() {
8832                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8833                None => Err(anyhow::anyhow!("repair has no winning branch")),
8834            };
8835            let why = match restored {
8836                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8837                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8838                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8839            };
8840            self.hold_merge_repair(i, why)?;
8841            return Ok(false);
8842        }
8843        let failure = if i >= self.state.config.graph.land_rounds
8844            || self.state.config.graph.review_rounds == 0
8845        {
8846            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8847        } else {
8848            None
8849        };
8850        if let Some(why) = failure {
8851            self.hold_merge_repair(i, why)?;
8852            return Ok(false);
8853        }
8854        if request.phase == Phase::Requested {
8855            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8856                Ok(pr) => pr,
8857                Err(e) => {
8858                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8859                    return Ok(false);
8860                }
8861            };
8862            self.state.merge_repairs[i].phase = Phase::Fixing;
8863            self.state.land_approval = None;
8864            self.state.land_armed_head = None;
8865            self.state.save()?;
8866            // Serialize the exact records, including finding ids/details,
8867            // reconsideration reasons, final votes and verification output.
8868            let evidence = serde_json::to_string_pretty(&request)?;
8869            let reason = format!(
8870                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8871            );
8872            let budget = self.state.config.graph.land_rounds;
8873            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8874            match result {
8875                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8876                Ok(land::Fixed::Failed(why)) => {
8877                    self.hold_merge_repair(
8878                        i,
8879                        format!(
8880                            "repair failed: {why}; inspect the branch and run an explicit review"
8881                        ),
8882                    )?;
8883                    return Ok(false);
8884                }
8885                Err(e) => {
8886                    self.hold_merge_repair(
8887                        i,
8888                        format!(
8889                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8890                        ),
8891                    )?;
8892                    return Ok(false);
8893                }
8894            }
8895            // Even no change gets a real new reviewer wave. The previous
8896            // evidence remains in the request; a former clean/concluded round
8897            // cannot short-circuit the new review or gate.
8898            self.begin_merge_repair_review(i)?;
8899        }
8900        self.review_loop().await?;
8901        self.gate().await?;
8902        if self.resource_halt()? || self.state.parked {
8903            return Ok(false);
8904        }
8905        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8906            != Some(RunStatus::Gating)
8907            || !self.state.gate_allows_progress()
8908        {
8909            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8910            return Ok(false);
8911        }
8912        // Review and gate may have added commits after the initial repair.
8913        // Publish exactly their final head to the existing PR, with a lease.
8914        let winner = self
8915            .state
8916            .winner()
8917            .cloned()
8918            .context("repair has no winning branch")?;
8919        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8920        if self.state.gate_head.as_deref() != Some(after.as_str())
8921            || !git::is_clean(&winner.worktree).await?
8922        {
8923            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8924            return Ok(false);
8925        }
8926        let expected = self
8927            .state
8928            .pr_pushed_head
8929            .as_deref()
8930            .unwrap_or(&request.head)
8931            .to_owned();
8932        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8933        // Push the checked commit itself: a local ref moving after the check
8934        // must not change which commit the forge receives.
8935        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8936        let refspec = format!("{after}:refs/heads/{public}");
8937        let push = git::git_raw(
8938            &winner.worktree,
8939            &["push", &lease, &self.state.config.merge.remote, &refspec],
8940        )
8941        .await;
8942        match push {
8943            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8944            Ok(out) => {
8945                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8946                return Ok(false);
8947            }
8948            Err(e) => {
8949                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8950                return Ok(false);
8951            }
8952        }
8953        self.state.merge_repairs[i].phase = Phase::Complete;
8954        self.state.status = RunStatus::Landing;
8955        self.state.event(
8956            "repair",
8957            "new review and gate completed; fresh owner merge decision required",
8958        );
8959        self.state.save()?;
8960        Ok(true)
8961    }
8962
8963    fn merge_repair_artifacts(&self) -> PathBuf {
8964        let artifacts = agent::artifacts_dir(&self.state.dir());
8965        match self
8966            .state
8967            .merge_repairs
8968            .last()
8969            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8970        {
8971            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8972            None => artifacts,
8973        }
8974    }
8975
8976    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8977        self.state.reviews.clear();
8978        self.state.gate.clear();
8979        self.state.gate_ran = false;
8980        self.state.gate_head = None;
8981        self.state.gate_fixes.clear();
8982        self.state.gate_repair_e2e = None;
8983        self.state.pre_gate.clear();
8984        self.state.pre_gate_commit = None;
8985        self.state.contested_handoff = None;
8986        self.state.land_reruns = None;
8987        self.state.land_approval = None;
8988        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8989        self.state.status = RunStatus::Reviewing;
8990        self.state.save()
8991    }
8992
8993    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8994        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8995        self.state.merge_repairs[index].failure = Some(why.clone());
8996        self.state.status = RunStatus::Blocked;
8997        self.state.event("repair", why);
8998        self.state.save()
8999    }
9000
9001    // -------------------------------------------------------------- helpers
9002
9003    /// Fetch or create a seat, keeping its conversation across nodes.
9004    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9005        if let Some(existing) = self.state.seats.get(key)
9006            && existing.agent == agent
9007        {
9008            return existing.clone();
9009        }
9010        // A seat that changes agent mints its session id from the agent too,
9011        // like a handover: the old agent's uuid is already taken by the CLI.
9012        let fresh = if self.state.seats.contains_key(key) {
9013            handover_seat(key, agent, self.state.next_seat_seed())
9014        } else {
9015            SeatState::new(key, agent, self.state.seed)
9016        };
9017        self.state.seats.insert(key.to_owned(), fresh.clone());
9018        fresh
9019    }
9020
9021    /// The agent now holding seat `key`: `spec`, unless a handover moved the
9022    /// seat to another roster agent, in which case that agent. Nodes that
9023    /// continue a seat's conversation (deliberation, the votes, a reviewer's
9024    /// reconsideration) must keep talking to whoever answered it, not slip
9025    /// back to the agent that failed it.
9026    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9027        match self.state.seats.get(key) {
9028            Some(s) if s.agent != spec.id => {
9029                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9030            }
9031            _ => spec,
9032        }
9033    }
9034
9035    /// A candidate rendered for judging, with the leak policy applied.
9036    fn view(&self, c: &Candidate) -> CandidateView {
9037        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9038            .unwrap_or_default();
9039        let (patch, _) = blind::sanitize_patch(
9040            &format!("candidate {} patch", c.label),
9041            &raw,
9042            &self.state.config.blind,
9043        );
9044        CandidateView {
9045            label: c.label,
9046            branch: c.branch.clone(),
9047            summary: c.summary.clone(),
9048            stat: c.stat.clone(),
9049            patch,
9050        }
9051    }
9052
9053    /// The full candidate set as prompt text, for seats with no live session.
9054    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9055        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9056        prompt::judge(
9057            "(see above)",
9058            &views,
9059            self.roles.judges.len(),
9060            base_short,
9061            "en",
9062        )
9063    }
9064
9065    /// The final-vote prompt with everything a seat that has no session of its
9066    /// own needs: the candidates, the seat's own ranking and reasons, and the
9067    /// anonymised deliberation it took part in (only when there was one, so a
9068    /// handed-over seat never sees more than the seat it replaces did). The
9069    /// `Final vote` heading stays first.
9070    fn vote_prompt_full(
9071        &self,
9072        j: usize,
9073        labels: &[char],
9074        language: &str,
9075        candidates: &[Candidate],
9076        base_short: &str,
9077    ) -> String {
9078        let mut text = format!(
9079            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9080            prompt::final_vote(labels, language),
9081            self.state.instruction,
9082            self.candidate_block(candidates, base_short)
9083        );
9084        if let Some(own) = self
9085            .state
9086            .judgements
9087            .get(j)
9088            .filter(|r| !r.ranking.is_empty())
9089        {
9090            let reasons = own
9091                .reasons
9092                .iter()
9093                .map(|(k, v)| format!("- {k}: {v}"))
9094                .collect::<Vec<_>>()
9095                .join("\n");
9096            text.push_str(&format!(
9097                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9098                own.ranking.iter().collect::<String>(),
9099                if reasons.is_empty() {
9100                    ""
9101                } else {
9102                    ", because:\n"
9103                }
9104            ));
9105        }
9106        if !self.state.deliberation.is_empty() {
9107            text.push_str("\n# What was argued before this vote\n");
9108            for t in self.transcript(&[], j) {
9109                text.push_str(&format!(
9110                    "\n## {}{}\n\n{}\n",
9111                    t.who,
9112                    if t.is_self { " (you)" } else { "" },
9113                    t.body.trim()
9114                ));
9115            }
9116        }
9117        text
9118    }
9119
9120    /// Anonymised transcript for judge `self_idx`.
9121    ///
9122    /// The initial rankings are always the opening statements. Seeding them
9123    /// only when no turn had been taken yet meant every judge after the first
9124    /// argued against a single voice instead of against the actual split — the
9125    /// disagreement is the information, so it is always on the table.
9126    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9127        let mut turns = Vec::new();
9128        for j in &self.state.judgements {
9129            if j.ranking.is_empty() {
9130                continue;
9131            }
9132            let reasons = j
9133                .reasons
9134                .iter()
9135                .map(|(k, v)| format!("- {k}: {v}"))
9136                .collect::<Vec<_>>()
9137                .join("\n");
9138            turns.push(Turn {
9139                who: format!("Judge {} (opening ranking)", j.judge),
9140                is_self: j.judge == self_idx + 1,
9141                body: format!(
9142                    "Ranked {}{}{reasons}",
9143                    j.ranking.iter().collect::<String>(),
9144                    if reasons.is_empty() {
9145                        ""
9146                    } else {
9147                        ", because:\n"
9148                    }
9149                ),
9150            });
9151        }
9152        for t in self
9153            .state
9154            .deliberation
9155            .iter()
9156            .flat_map(|r| r.turns.iter())
9157            .chain(current)
9158        {
9159            turns.push(Turn {
9160                who: format!("Judge {}", t.judge),
9161                is_self: t.judge == self_idx + 1,
9162                body: t.body.clone(),
9163            });
9164        }
9165        turns
9166    }
9167}
9168
9169/// Does this seat still hold the context a follow-up prompt would rely on?
9170fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9171    agent::has_session(spec.kind, seat, sessions)
9172}
9173
9174/// The next entry in `roster` after `start`, never wrapping back to the
9175/// front, whose id is not in `tried` yet.
9176///
9177/// Starts one past `start` rather than at the front of `roster`: `start` is
9178/// the seat's own original position, and a seat whose candidate slot already
9179/// sits on the roster's second entry must fall through to the third next, not
9180/// restart at the first — which is very likely a different candidate's own
9181/// agent already. Never wraps back past `start`, for the same reason: an
9182/// entry earlier in the roster than the seat's own position is almost
9183/// certainly some *other* candidate slot's own agent, and once the tail of
9184/// the roster is exhausted there are no more untried agents for *this* seat
9185/// to fall through to — the caller's fallback chain ends there, exactly as
9186/// "no further untried agents remain in the list for that seat" asks for.
9187///
9188/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
9189/// the same id twice (an operator's `roles.implementers` typo, or a
9190/// `[[agents]]` list reused across roles) must not let
9191/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
9192/// over `roster` either finds an untried id or runs out, so this always
9193/// terminates regardless of duplicates.
9194fn next_untried_in_roster<'a>(
9195    roster: &'a [AgentSpec],
9196    start: usize,
9197    tried: &BTreeSet<String>,
9198) -> Option<&'a AgentSpec> {
9199    roster
9200        .get(start + 1..)?
9201        .iter()
9202        .find(|s| !tried.contains(&s.id))
9203}
9204
9205/// The successor for a seat, shared by all four seat kinds (implement, judge,
9206/// review, advise). `others` holds the ids that *currently* occupy the other
9207/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9208/// sees them), so roster entries beyond the seat count act as spares: a failed
9209/// seat goes to an agent no other seat holds whenever the roster permits.
9210///
9211/// `carried` is `Some` only for the review loop's carried failure history.
9212/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9213/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9214/// whole roster: untried and not another seat's occupant; (3) the plain walk
9215/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9216/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9217/// a better outcome than an empty seat, and it keeps the answer to "is there
9218/// a successor at all" exactly what it was before occupants were considered,
9219/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9220/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9221/// carried failure is a cheaper bet than doubling an agent on the panel, and
9222/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9223/// the same round are handled one at a time, so a seat later in the batch
9224/// sees an earlier one's new occupant but not yet its own freed agent.
9225fn pick_successor<'a>(
9226    roster: &'a [AgentSpec],
9227    start: usize,
9228    tried: &BTreeSet<String>,
9229    carried: Option<&BTreeSet<String>>,
9230    others: &BTreeSet<String>,
9231) -> Option<&'a AgentSpec> {
9232    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9233    roster
9234        .get(start + 1..)
9235        .and_then(|tail| {
9236            tail.iter()
9237                .filter(free)
9238                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9239        })
9240        .or_else(|| {
9241            carried
9242                .is_some()
9243                .then(|| roster.iter().find(free))
9244                .flatten()
9245        })
9246        .or_else(|| match carried {
9247            Some(c) => next_for_seat(roster, start, tried, c),
9248            None => next_untried_in_roster(roster, start, tried),
9249        })
9250}
9251
9252/// The next agent for a seat that carries its failure history across rounds
9253/// (the review loop). `round_tried` is this round's own bound and starts
9254/// empty every round; `carried_failed` only decides priority.
9255///
9256/// First: an id walking forward from `start`, never wrapping, that is neither
9257/// tried this round nor failed in an earlier one. Only when that is exhausted
9258/// does it rescue: the first roster id (in roster order, so this one *does*
9259/// look before `start`) not yet tried this round, which is by then a carried
9260/// failure. Each id is rescued at most once per round, so it cannot loop.
9261fn next_for_seat<'a>(
9262    roster: &'a [AgentSpec],
9263    start: usize,
9264    round_tried: &BTreeSet<String>,
9265    carried_failed: &BTreeSet<String>,
9266) -> Option<&'a AgentSpec> {
9267    roster
9268        .get(start + 1..)?
9269        .iter()
9270        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9271        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9272}
9273
9274/// Where a reviewer seat starts a round: the agent that last answered it when
9275/// it is still on the roster and not marked failed, else the spec's own agent
9276/// unless it failed, else the next roster agent that has not failed, else the
9277/// spec's own agent again (the whole roster failed). Ids no longer on the
9278/// roster are ignored. An empty roster has no handover, so the spec stands.
9279fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9280    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9281        return spec;
9282    };
9283    let ok = |id: &str| !h.failed.contains(id);
9284    if let Some(last) = h.last_ok.as_deref()
9285        && ok(last)
9286        && let Some(s) = roster.iter().find(|s| s.id == last)
9287    {
9288        return s.clone();
9289    }
9290    if ok(&spec.id) {
9291        return spec;
9292    }
9293    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9294    roster
9295        .get(from + 1..)
9296        .into_iter()
9297        .flatten()
9298        .chain(roster.iter())
9299        .find(|s| ok(&s.id))
9300        .cloned()
9301        .unwrap_or(spec)
9302}
9303
9304/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9305/// seed so a CLI that mints its session id up front (`--session-id`) never
9306/// reuses the uuid the previous agent already opened under the same seat key.
9307pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9308    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9309}
9310
9311/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9312/// so the classifier below and the code that builds the message cannot drift.
9313const TIMED_OUT: &str = "timed out";
9314
9315impl FailClass {
9316    /// `None` for an answer; otherwise how the turn failed.
9317    fn of(out: &AgentOutcome) -> Option<Self> {
9318        match out {
9319            AgentOutcome::Ok(_) => None,
9320            AgentOutcome::Quota(_) => Some(Self::Quota),
9321            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9322            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9323            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9324            // Not a failure of the agent. Callers check for it first; `None`
9325            // here means "no class to hand over on".
9326            AgentOutcome::ResourceBlocked(_) => None,
9327        }
9328    }
9329
9330    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9331    fn stem_word(&self) -> &'static str {
9332        match self {
9333            Self::Quota => "quota",
9334            _ => "handover",
9335        }
9336    }
9337}
9338
9339/// The message's first line with its variable parts removed — digit runs and
9340/// path-like tokens — so "exit code 2" and "exit code 7" read
9341/// as one kind of failure.
9342fn failure_signature(msg: &str) -> String {
9343    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9344    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9345    // the historical machine key while the process diagnosis uses readable prose.
9346    if let Some(code) = line
9347        .strip_prefix("process exited with no usable output (exit code ")
9348        .and_then(|s| s.strip_suffix(')'))
9349        .and_then(|s| s.parse::<i32>().ok())
9350    {
9351        line = format!("exited with some({code}) and no usable output");
9352    } else if matches!(
9353        line.as_str(),
9354        "process exited with no usable output (no exit code reported)"
9355            | "process exited with no usable output (exit code unavailable)"
9356    ) {
9357        line = "exited with none and no usable output".to_owned();
9358    }
9359    let mut out = Vec::new();
9360    for word in line.split_whitespace() {
9361        if word.contains('/') || word.contains('\\') {
9362            out.push("<path>".to_owned());
9363            continue;
9364        }
9365        let mut w = String::new();
9366        let mut in_digits = false;
9367        for c in word.chars() {
9368            if c.is_ascii_digit() {
9369                if !in_digits {
9370                    w.push('#');
9371                }
9372                in_digits = true;
9373            } else {
9374                in_digits = false;
9375                w.push(c);
9376            }
9377        }
9378        out.push(w);
9379    }
9380    out.join(" ").chars().take(120).collect()
9381}
9382
9383/// Whether a seat that just failed with `cur` may go to the next roster agent.
9384/// A quota or a timeout always may. Any other failure may not when the agent
9385/// before it failed the same way: an error the prompt causes would otherwise
9386/// walk the whole roster. `prev` is the class of the immediately preceding
9387/// agent's failure, so a quota or timeout in between breaks the run of
9388/// identical failures by itself.
9389fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9390    match cur {
9391        FailClass::Quota | FailClass::Timeout => true,
9392        FailClass::Other(_) => prev != Some(cur),
9393    }
9394}
9395
9396/// A short human reason for a failed outcome, for the handover record.
9397fn fail_reason(out: &AgentOutcome) -> String {
9398    match out {
9399        AgentOutcome::Ok(_) => String::new(),
9400        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9401        AgentOutcome::Dropped(o) => format!(
9402            "the CLI dropped the stream ({})",
9403            o.dropped
9404                .as_ref()
9405                .map(|d| d.why.as_str())
9406                .unwrap_or("it ended without delivering its answer")
9407        ),
9408        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9409            e.lines().next().unwrap_or("").chars().take(160).collect()
9410        }
9411    }
9412}
9413
9414/// Note one handover in the run: the structured record and, in the timeline,
9415/// the sentence a person reads. A quota keeps the wording it always had.
9416pub(crate) fn record_handover(
9417    state: &mut RunState,
9418    node: &str,
9419    seat: &str,
9420    from: &str,
9421    to: &str,
9422    class: &FailClass,
9423    reason: &str,
9424) {
9425    let message = if *class == FailClass::Quota {
9426        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9427    } else {
9428        format!("{seat}: handed over {from} -> {to} ({reason})")
9429    };
9430    state.event(node, message);
9431    state.handovers.push(Handover {
9432        at: Timestamp::now(),
9433        node: node.to_owned(),
9434        seat: seat.to_owned(),
9435        from: from.to_owned(),
9436        to: to.to_owned(),
9437        reason: reason.to_owned(),
9438    });
9439}
9440
9441/// Did this reply report running a command whose own CLI never confirmed an
9442/// exit status?
9443///
9444/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9445/// command *finished* (see that type's own doc), so this can only be `true`
9446/// for a command whose completion event carried no readable exit code — not
9447/// for one that simply is not mentioned at all. That is the one signal this
9448/// crate can read, from the same record `state.jobs` renders, about a reply
9449/// standing next to work its own CLI cannot vouch for finishing; it is
9450/// deliberately not a check on the exit code's *value* (a fixer legitimately
9451/// runs a command that fails mid-iteration before it succeeds) and not a
9452/// guess at a command still running in the background (which emits no event
9453/// at all, and so leaves no evidence here to find).
9454fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9455    commands.iter().any(|c| c.exit_code.is_none())
9456}
9457
9458/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9459/// trusted as a verified no-op — the adoption guard's own text-level half.
9460///
9461/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9462/// timed out): a marker only earns the benefit of the doubt from a turn the
9463/// CLI itself vouches for finishing properly, the same house style
9464/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9465/// *fix* report to for `commands`. A candidate that timed out, exited
9466/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9467/// is, whatever prose it wrote — this returns `None` before it ever looks at
9468/// `text`. The remaining guards (the tree really is empty, the evidence is
9469/// non-empty) are the caller's: this only reads what the reply *claimed*.
9470fn verified_noop_claim(
9471    usable: bool,
9472    commands: &[agent::CommandEvidence],
9473    text: &str,
9474) -> Option<String> {
9475    (usable && !has_unconfirmed_command(commands))
9476        .then(|| verdict::verified_noop(text))
9477        .flatten()
9478}
9479
9480fn short(commit: &str) -> String {
9481    commit.chars().take(7).collect()
9482}
9483
9484fn make_executable(path: &Path) -> Result<()> {
9485    #[cfg(unix)]
9486    {
9487        use std::os::unix::fs::PermissionsExt as _;
9488        let mut perms = std::fs::metadata(path)?.permissions();
9489        perms.set_mode(0o755);
9490        std::fs::set_permissions(path, perms)?;
9491    }
9492    #[cfg(not(unix))]
9493    {
9494        let _ = path;
9495    }
9496    Ok(())
9497}
9498
9499/// What every seat in one batch shares: where the answers are attributed, the
9500/// prompt overlay they inherit, and the build cache they are told to use.
9501///
9502/// A struct rather than four more parameters: `wave` also needs the run's
9503/// state (to record who is answering right now) and the attempt number, and
9504/// eight positional arguments is both unreadable and a clippy error.
9505struct WaveCtx<'a> {
9506    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9507    /// paid for it.
9508    run: &'a str,
9509    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9510    node: &'a str,
9511    prompts: &'a Prompts,
9512    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9513    cache: Option<&'a Path>,
9514    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9515    /// `JobRecord::round`. `None` for every other node.
9516    round: Option<usize>,
9517    /// Carry each seat's failed-agent history across waves (the review loop
9518    /// only): start-of-round priority and handover choice read
9519    /// [`RunState::seat_history`], and every answer or failure writes it.
9520    carry_seats: bool,
9521    /// The run's park / interrupt requests, so a wait for a shared resource
9522    /// can be abandoned the moment the operator asks to park.
9523    pauses: Option<(&'a Pause, &'a Pause)>,
9524}
9525
9526/// Shared scheduling policy for operations and spawned seat invocations.
9527fn repository_wait(
9528    state: &RunState,
9529    pauses: Option<(&Pause, &Pause)>,
9530) -> crate::repo_lock::WaitPolicy {
9531    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9532    let inherited = crate::repo_lock::WaitPolicy::current();
9533    crate::repo_lock::WaitPolicy::new(
9534        Duration::from_secs(state.config.graph.timeout_resource_wait),
9535        move || {
9536            inherited.cancelled()
9537                || pauses
9538                    .as_ref()
9539                    .is_some_and(|(a, b)| a.parked() || b.parked())
9540        },
9541    )
9542}
9543
9544fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9545    if error.is::<crate::repo_lock::ResourceBlocked>() {
9546        state.resource_block = Some(crate::run::ResourceBlock {
9547            cause: crate::run::ResourceBlockCause::Other,
9548            node: "repository".into(),
9549            kind: if error.to_string().contains("disk pressure") {
9550                "disk pressure"
9551            } else {
9552                "repository coordination"
9553            }
9554            .into(),
9555            detail: error.to_string(),
9556            at: Timestamp::now(),
9557        });
9558        state.event("resource", error.to_string());
9559    }
9560}
9561
9562async fn coordinate_operation(
9563    state: &mut RunState,
9564    cwd: &Path,
9565    wait: &crate::repo_lock::WaitPolicy,
9566) -> Result<Option<crate::repo_lock::SeatLease>> {
9567    let result =
9568        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9569            .await;
9570    if let Err(e) = &result {
9571        record_repository_block(state, e);
9572    }
9573    result
9574}
9575
9576/// Run one job, honouring the parallelism budget.
9577async fn run_one(
9578    job: SeatJob,
9579    sem: Arc<Semaphore>,
9580    ctx: &WaveCtx<'_>,
9581    state: &mut RunState,
9582    attempt: usize,
9583) -> (SeatState, AgentOutcome) {
9584    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9585        .await
9586        .pop()
9587        .expect("one job in, one result out");
9588    (seat, out)
9589}
9590
9591/// Run every job concurrently, capped by the semaphore, preserving order.
9592///
9593/// Every seat in the batch is recorded into [`RunState::active`] before the
9594/// wave starts and cleared as each answer lands, so the run's own record says
9595/// who is still being waited on rather than only who finished.
9596async fn wave(
9597    jobs: Vec<SeatJob>,
9598    sem: Arc<Semaphore>,
9599    ctx: &WaveCtx<'_>,
9600    state: &mut RunState,
9601    attempt: usize,
9602) -> Vec<(usize, SeatState, AgentOutcome)> {
9603    let WaveCtx {
9604        run,
9605        node,
9606        prompts,
9607        cache,
9608        round,
9609        carry_seats: _,
9610        pauses,
9611    } = *ctx;
9612    for job in &jobs {
9613        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9614    }
9615    if let Err(e) = state.save() {
9616        // A failed persist of "who is answering right now" must not abort the
9617        // wave: the seats are already being asked, and the alternative is
9618        // losing the answers to save a status line nobody may even be
9619        // watching.
9620        tracing::warn!("could not persist in-progress seats: {e:#}");
9621    }
9622    // Hold the shared build cache's lease for the whole batch, not per job:
9623    // several candidates (an implement wave) or a fixer legitimately share
9624    // one cache concurrently within this run, and that stays untouched — a
9625    // single lease taken once for the whole wave and released once it is
9626    // done is what stops a *different* borrower (another run's own wave, its
9627    // e2e/gate, a human's `magi review`) from interleaving a build into the
9628    // same directory while this one is in flight. Best-effort, not
9629    // all-or-nothing: a wave that cannot get the lease within its own
9630    // longest job's budget still runs — an hour of paid implementer calls is
9631    // not thrown away over cache contention — but every write-allowed seat
9632    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9633    // below), the same fallback a read-only seat always gets, rather than
9634    // building into a directory this run was never granted. The identity
9635    // record is still invalidated below either way, so the next tracked
9636    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9637    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9638    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9639    // never borrowed from a job's agent budget (and never charged to it either,
9640    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9641    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9642    let cache_guard = if let Some(cache_dir) = cache {
9643        if jobs_had_a_writer {
9644            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9645            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9646            match acquire_cache_lease(
9647                state,
9648                cache_dir,
9649                &owner,
9650                resource_limit,
9651                node,
9652                node,
9653                &cancel,
9654            )
9655            .await
9656            {
9657                Ok(Some(g)) => Some(g),
9658                // Parking: leave without spawning anything. The run parks at
9659                // the next boundary and the seats are asked afresh on resume.
9660                Ok(None) => {
9661                    if let Some(block) = &mut state.resource_block {
9662                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9663                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9664                        };
9665                    }
9666                    return block_wave(
9667                        jobs,
9668                        state,
9669                        node,
9670                        attempt,
9671                        "parked while waiting for the shared build cache",
9672                    );
9673                }
9674                // An unreadable lease is not contention: keep the old
9675                // fallback (no shared cache for this wave's seats).
9676                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9677                Err(e) => {
9678                    // Writing seats may not build into a cache this run does
9679                    // not hold, and an agent that cannot be given one is not
9680                    // asked at all: the run stops resumably instead.
9681                    let why = format!("{e:#}");
9682                    state.event(
9683                        "resource",
9684                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9685                    );
9686                    return block_wave(jobs, state, node, attempt, &why);
9687                }
9688            }
9689        } else {
9690            None
9691        }
9692    } else {
9693        None
9694    };
9695    // Every job keeps its whole budget: the wait above was not the agent's.
9696    // Restart the seats' clocks so "started" means "being spawned now".
9697    state.seats_spawning(node, attempt);
9698    if let Err(e) = state.save() {
9699        tracing::warn!("could not persist the start of a wave: {e:#}");
9700    }
9701    // Queueing behind this wave's own seats for a parallelism slot is normal
9702    // (each of them is bounded by its own agent timeout); only a wait beyond
9703    // all of them plus the resource limit means the slots never came back.
9704    let slot_limit = resource_limit
9705        + jobs
9706            .iter()
9707            .map(|j| j.timeout)
9708            .max()
9709            .unwrap_or_default()
9710            .saturating_mul(jobs.len() as u32);
9711    let mut set = tokio::task::JoinSet::new();
9712    // Seats announce themselves here the moment they hold a parallelism
9713    // permit, i.e. just before their CLI is spawned.
9714    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9715    let overlay = prompts.overlay(node);
9716    let repository_wait = repository_wait(state, pauses);
9717    for (i, mut job) in jobs.into_iter().enumerate() {
9718        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9719        if cache.is_some() {
9720            job.prompt.push('\n');
9721            job.prompt
9722                .push_str(&prompt::build_cache_note(node, job.allow_write));
9723        }
9724        let sem = Arc::clone(&sem);
9725        let run = run.to_owned();
9726        let node = node.to_owned();
9727        // Only implementers were told about the task's attachments, so only
9728        // their seats get the directory widened for reading.
9729        let attachments = if node == "implement" {
9730            state.attachments.clone()
9731        } else {
9732            Vec::new()
9733        };
9734        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9735        // `prompt::build_cache_note`'s doc for why setting it anyway is
9736        // exactly how a sandboxed reviewer's write refusal got reported as a
9737        // defect in the patch, not a property of its own seat. And a
9738        // write-allowed one is handed it only when the lease above was
9739        // actually acquired: a wave that could not get it (`cache_guard` is
9740        // `None`, see its own comment) must not send seats to build into a
9741        // directory this run does not hold - that is the exact concurrent,
9742        // unmanaged-write race this module exists to prevent, not something
9743        // "proceeding anyway" is allowed to reintroduce.
9744        let cache = cache
9745            .filter(|_| job.allow_write && cache_guard.is_some())
9746            .map(Path::to_path_buf);
9747        let spawned_tx = spawned_tx.clone();
9748        let repository_wait = repository_wait.clone();
9749        let coordination = state.config.coordination.clone();
9750        let fetched_base = state.fetched_base.clone();
9751        set.spawn(async move {
9752            // A wait for a parallelism slot is scheduling, not the agent: it
9753            // is bounded by its own limit and ends as a resource block, never
9754            // as a timeout of the agent that was not started.
9755            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9756                Ok(p) => p,
9757                Err(_) => {
9758                    return (
9759                        i,
9760                        job.seat,
9761                        AgentOutcome::ResourceBlocked(format!(
9762                            "no parallel slot became free within {}s",
9763                            slot_limit.as_secs()
9764                        )),
9765                        false,
9766                    );
9767                }
9768            };
9769            spawned_tx.send(job.seat.key.clone()).ok();
9770            let mut seat = job.seat;
9771            let out = repository_wait
9772                .scope(agent::invoke(
9773                    &job.spec,
9774                    &mut seat,
9775                    &Invocation {
9776                        coordination: coordination.as_ref(),
9777                        fetched_base: fetched_base.as_ref(),
9778                        cwd: &job.cwd,
9779                        prompt: &job.prompt,
9780                        timeout: job.timeout,
9781                        allow_write: job.allow_write,
9782                        sessions: job.sessions,
9783                        artifacts: &job.artifacts,
9784                        stem: &job.stem,
9785                        run: &run,
9786                        node: &node,
9787                        cache_dir: cache.as_deref(),
9788                        attachments: &attachments,
9789                        writable: &[],
9790                    },
9791                ))
9792                .await;
9793            let interrupted = out
9794                .as_ref()
9795                .err()
9796                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9797            let out = match out {
9798                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9799                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9800                // Billed work the CLI failed to hand over is not an ordinary
9801                // failure, but its text is the CLI's raw error JSON, not an
9802                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9803                // read it as one by forgetting to check. `usable()` is always
9804                // false here (dropped implies an empty response), so this has
9805                // to be checked before the catch-all `Failed` below or the
9806                // one shape this exists for is lost with the rest.
9807                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9808                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9809                Ok(o) => AgentOutcome::Failed(format!(
9810                    "process exited with no usable output ({})",
9811                    crate::notices::exit_status(o.exit_code)
9812                )),
9813                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9814                    AgentOutcome::ResourceBlocked(e.to_string())
9815                }
9816                Err(e) => AgentOutcome::Failed(e.to_string()),
9817            };
9818            (i, seat, out, interrupted)
9819        });
9820    }
9821    drop(spawned_tx);
9822    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9823    loop {
9824        let joined = tokio::select! {
9825            Some(key) = spawned_rx.recv() => {
9826                // The clock of this seat starts now, not when it queued.
9827                state.seat_spawned(&key);
9828                if let Err(e) = state.save() {
9829                    tracing::warn!("could not persist a seat's start: {e:#}");
9830                }
9831                continue;
9832            }
9833            joined = set.join_next() => joined,
9834        };
9835        let Some(joined) = joined else {
9836            break;
9837        };
9838        let (i, seat, out, interrupted) = match joined {
9839            Ok(v) => v,
9840            // No seat to clear: a panicked task never reported which one it
9841            // was. The defensive sweep below this loop is what stops that
9842            // seat's `active` entry from surviving forever.
9843            Err(e) => {
9844                tracing::error!("agent task panicked: {e}");
9845                continue;
9846            }
9847        };
9848        state.seat_finished(&seat.key);
9849        if interrupted
9850            && node == "implement"
9851            && let Some(label) = state
9852                .candidates
9853                .iter()
9854                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9855                .map(|candidate| candidate.label)
9856        {
9857            state.candidate_preservation.insert(
9858                label,
9859                "interrupted implementation awaiting persistence".into(),
9860            );
9861            state.interrupted_implementations.insert(label);
9862        }
9863        if let AgentOutcome::ResourceBlocked(why) = &out
9864            && state.resource_block.is_none()
9865        {
9866            state.resource_block = Some(crate::run::ResourceBlock {
9867                cause: crate::run::ResourceBlockCause::Other,
9868                node: node.to_owned(),
9869                kind: if why.contains("disk pressure") {
9870                    "disk pressure"
9871                } else if why.contains("repository coordination") {
9872                    "repository coordination"
9873                } else {
9874                    "parallel slot"
9875                }
9876                .to_owned(),
9877                detail: why.clone(),
9878                at: Timestamp::now(),
9879            });
9880            let activity = if interrupted {
9881                "agent execution interrupted"
9882            } else {
9883                "no agent was started"
9884            };
9885            state.event("resource", format!("{node}: {activity} - {why}"));
9886        }
9887        record_jobs(state, node, round, &seat.key, &out);
9888        if let Err(e) = state.save() {
9889            tracing::warn!("could not persist a seat's completion: {e:#}");
9890        }
9891        if collected.len() <= i {
9892            collected.resize_with(i + 1, || None);
9893        }
9894        collected[i] = Some((i, seat, out));
9895    }
9896    // Belt-and-braces for the panic branch above: every seat this exact batch
9897    // started shares this `(node, attempt)` pair, and every seat that finished
9898    // normally already cleared itself, so anything left tagged with it here
9899    // can only be a panicked task's leftover. Cleared unconditionally rather
9900    // than left to read as still answering forever.
9901    if state
9902        .active
9903        .values()
9904        .any(|a| a.node == node && a.attempt == attempt)
9905    {
9906        state
9907            .active
9908            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9909        if let Err(e) = state.save() {
9910            tracing::warn!("could not persist the end of a wave: {e:#}");
9911        }
9912    }
9913    // Whether or not the lease above was actually held, several worktrees
9914    // may just have built into the cache with nothing here able to name one
9915    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9916    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9917    // clean it might not have strictly needed; trusting a stale match would
9918    // cost it a wrong answer.
9919    if let Some(cache_dir) = cache
9920        && jobs_had_a_writer
9921    {
9922        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9923    }
9924    if let Some(guard) = cache_guard {
9925        guard.release();
9926    }
9927    collected.into_iter().flatten().collect()
9928}
9929
9930/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9931/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9932/// this is data collection, not the fix-specific completion contract in
9933/// [`Runner::continue_fix_report`], and applies regardless of which node
9934/// asked.
9935///
9936/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9937/// evidence from; `Failed` does not, and correctly contributes nothing — a
9938/// timeout or crash is not itself evidence about a command the seat may have
9939/// started.
9940fn record_jobs(
9941    state: &mut RunState,
9942    node: &str,
9943    round: Option<usize>,
9944    seat: &str,
9945    out: &AgentOutcome,
9946) {
9947    let commands: &[agent::CommandEvidence] = match out {
9948        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9949        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9950    };
9951    let checked_at = Timestamp::now();
9952    for c in commands {
9953        state.jobs.push(JobRecord {
9954            node: node.to_owned(),
9955            round,
9956            seat: seat.to_owned(),
9957            id: c.id.clone(),
9958            description: c.description.clone(),
9959            checked_at,
9960            status: match c.exit_code {
9961                Some(0) => JobStatus::Completed,
9962                Some(_) => JobStatus::Failed,
9963                None => JobStatus::Unknown,
9964            },
9965            exit_code: c.exit_code,
9966            result_summary: c.result_summary.clone(),
9967            source: c.source.clone(),
9968        });
9969    }
9970}
9971
9972/// Is a review round clean, given how many reviewer seats answered against
9973/// how many the round expected?
9974///
9975/// A seat that never answered (timeout, crash, unparsable output) is not a
9976/// seat that read the patch and found nothing — treating it as such is
9977/// exactly the bug this function exists to close. Under the default `block`
9978/// policy a missing seat can never be clean; `warn` still requires the seats
9979/// that *did* answer to have found nothing blocking and verification to be
9980/// green.
9981///
9982/// `quota_missing` narrows that `block` default for exactly one cause of
9983/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9984/// a session limit lifts by the very next round buys nothing — the seat is
9985/// asked again with the same quota — so once every missing seat is accounted
9986/// for by a quota loss (and at least one seat *did* answer, so a decision has
9987/// something to rest on) the round is decided on the panel that could answer,
9988/// same as `warn` would. A panel that lost every seat to quota is not
9989/// decided here: `answered == 0` falls through to the existing `block`
9990/// fallback so a fully collapsed panel still waits rather than landing on no
9991/// review at all.
9992fn round_is_clean(
9993    blocking: usize,
9994    e2e_ok: bool,
9995    answered: usize,
9996    expected: usize,
9997    quota_missing: usize,
9998    policy: IncompleteReviewPolicy,
9999) -> bool {
10000    if blocking != 0 || !e2e_ok {
10001        return false;
10002    }
10003    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10004        return true;
10005    }
10006    answered > 0 && expected - answered <= quota_missing
10007}
10008
10009/// The review loop's own conclusion, derived entirely from its persisted
10010/// round records and the round budget that produced them — never from
10011/// `status`, so a reentry (or `gate`/`merge` reading it independently)
10012/// recomputes the identical answer regardless of what an earlier node in the
10013/// same walk, or a previous walk, did to `status`.
10014///
10015/// `None` while more rounds remain to try, including when review never ran
10016/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
10017/// gone clean, or the budget is spent, or the tree has stopped moving (see
10018/// [`STAGNANT_LIMIT`]), the answer is one of two things:
10019///
10020/// - An incomplete panel that raised nothing is missing input, not a
10021///   verified tree — never a hand-off candidate, whatever verification said
10022///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
10023/// - Otherwise, green e2e on the last round hands off (see
10024///   [`Runner::stop_reviewing`]); red e2e blocks.
10025///
10026/// A last round whose own verification is still `ResourceBlocked` — magi
10027/// itself never got a command to run, not evidence the patch is broken —
10028/// is neither: this returns `None` for it too, the same as "more rounds
10029/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
10030/// handling of that shape) instead of this cheap recomputation guessing a
10031/// verdict a real attempt never produced.
10032pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10033    if max_rounds == 0
10034        || reviews.last().is_some_and(|r| {
10035            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10036        })
10037    {
10038        return Some(RunStatus::Gating);
10039    }
10040    let last = reviews.last()?;
10041    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10042    if reviews.len() < max_rounds && !stagnant {
10043        return None;
10044    }
10045    if last.incomplete() && last.blocking == 0 {
10046        return Some(RunStatus::Blocked);
10047    }
10048    if last.e2e_status() == E2eStatus::ResourceBlocked
10049        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10050    {
10051        return None;
10052    }
10053    Some(if last.e2e.iter().all(CommandOutcome::ok) {
10054        RunStatus::Gating
10055    } else {
10056        RunStatus::Blocked
10057    })
10058}
10059
10060/// How long a re-ask may take, given the budget the first attempt had.
10061///
10062/// A `nudged` retry is a request to restate an answer the seat has already
10063/// worked out: it carries no new work, so it does not deserve the original
10064/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
10065/// 133 seconds while a third sat for over ten minutes on a resumed session
10066/// holding 410 KB of prior output - and because the retry had inherited the
10067/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
10068/// judging round whose other seats were long finished.
10069///
10070/// A quarter of the budget, with a floor so that a deliberately short timeout
10071/// does not collapse to nothing. A retry that re-sends the whole prompt
10072/// (because the seat kept no context) is the original job again, and keeps the
10073/// original budget.
10074pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10075    if nudged {
10076        (full / 4).max(Duration::from_secs(120)).min(full)
10077    } else {
10078        full
10079    }
10080}
10081
10082/// Run a wave and parse each reply, re-asking the seats whose reply was
10083/// unusable.
10084///
10085/// The re-ask is a nudge rather than the whole prompt again when the seat still
10086/// holds its conversation, which is the difference between a cheap retry and
10087/// paying for the entire candidate set twice.
10088///
10089/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
10090/// successor in `roster` is handed to it instead of being re-asked: the
10091/// handover is the retry. A seat with no successor left (a single-agent
10092/// roster, the roster's tail) is nudged as before, up to `retries` times. So
10093/// the asks to one seat in one node number at most
10094/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
10095/// is asked once (a dropped stream is nudged first), and only the last agent
10096/// of the chain gets the `retries` same-agent nudges. Each roster agent is
10097/// tried at most once per seat, walking forward from the seat's own position and never wrapping
10098/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
10099/// other failure stops the chain when the previous agent failed the same way
10100/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
10101/// [`has_context`] is false and the job's own full prompt and full budget are
10102/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
10103/// intermediate ones are not losses) and is returned as a failure like any
10104/// other absent seat — the caller decides whether the panel still has a
10105/// quorum. An empty `roster` disables handover: failures are nudged as they
10106/// always were, and a quota is simply lost. A reply that fails to parse or
10107/// validate is the prompt's doing and is only ever nudged, never handed over.
10108///
10109/// The returned [`SeatState`] names the agent that answered (or tried last).
10110#[allow(clippy::too_many_arguments)]
10111async fn ask_json_wave<T>(
10112    jobs: Vec<SeatJob>,
10113    sem: Arc<Semaphore>,
10114    retries: usize,
10115    roster: &[AgentSpec],
10116    ctx: &WaveCtx<'_>,
10117    losses: &mut Vec<QuotaLoss>,
10118    state: &mut RunState,
10119    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10120) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10121where
10122    T: serde::de::DeserializeOwned + Send + 'static,
10123{
10124    ask_wave_with(
10125        jobs,
10126        sem,
10127        retries,
10128        roster,
10129        ctx,
10130        losses,
10131        state,
10132        &|text: &str| {
10133            let v = verdict::extract_json::<T>(text)?;
10134            validate(&v)?;
10135            Ok(v)
10136        },
10137    )
10138    .await
10139}
10140
10141/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
10142/// a node whose answer is prose (deliberation) shares the same handover,
10143/// failure classification, quota bookkeeping and bounds instead of a copy.
10144///
10145/// A seat handed to another roster agent is sent the job's `handover` prompt
10146/// (when it has one) rather than `prompt`: the new agent has no session, so a
10147/// resume-style prompt would be incomplete. That holds for the handover ask
10148/// and for every nudge to that agent whose `has_context` is false.
10149#[allow(clippy::too_many_arguments)]
10150async fn ask_wave_with<T>(
10151    jobs: Vec<SeatJob>,
10152    sem: Arc<Semaphore>,
10153    retries: usize,
10154    roster: &[AgentSpec],
10155    ctx: &WaveCtx<'_>,
10156    losses: &mut Vec<QuotaLoss>,
10157    state: &mut RunState,
10158    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10159) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10160where
10161    T: Send + 'static,
10162{
10163    let n = jobs.len();
10164    let originals: Vec<SeatJob> = jobs;
10165    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10166    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10167    // Nudges each seat's *current* agent has taken — 0 for a first-ask
10168    // answer, N once it has gone through N nudges. Read back once this
10169    // returns, so a caller building a history record (`ReviewRecord`) can
10170    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
10171    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
10172    // field's own doc.
10173    let mut nudges: Vec<usize> = vec![0; n];
10174    // The agent now occupying each seat, the ids it has already been through,
10175    // where in the roster the walk began, the class of the last failure, and
10176    // the stem word of a handover not yet asked (full prompt, full budget).
10177    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10178    let mut tried: Vec<BTreeSet<String>> = specs
10179        .iter()
10180        .map(|s| BTreeSet::from([s.id.clone()]))
10181        .collect();
10182    let starts: Vec<usize> = specs
10183        .iter()
10184        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10185        .collect();
10186    let carry = ctx.carry_seats && !roster.is_empty();
10187    // Carried across rounds: ids that failed the seat earlier, and how the
10188    // last failure went (so a repeat of it is not handed over again).
10189    let carried: Vec<BTreeSet<String>> = originals
10190        .iter()
10191        .map(|j| {
10192            state
10193                .seat_history
10194                .get(&j.seat.key)
10195                .filter(|_| carry)
10196                .map(|h| h.failed.clone())
10197                .unwrap_or_default()
10198        })
10199        .collect();
10200    let mut prev: Vec<Option<FailClass>> = originals
10201        .iter()
10202        .map(|j| {
10203            state
10204                .seat_history
10205                .get(&j.seat.key)
10206                .filter(|_| carry)
10207                .and_then(|h| h.last_fail.clone())
10208        })
10209        .collect();
10210    let next_agent =
10211        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10212            let others: BTreeSet<String> = specs
10213                .iter()
10214                .enumerate()
10215                .filter(|(j, _)| *j != i)
10216                .map(|(_, s)| s.id.clone())
10217                .collect();
10218            pick_successor(
10219                roster,
10220                starts[i],
10221                tried,
10222                carry.then(|| &carried[i]),
10223                &others,
10224            )
10225            .cloned()
10226        };
10227    let mut fresh: Vec<Option<String>> = vec![None; n];
10228    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10229    let mut pending: Vec<usize> = (0..n).collect();
10230
10231    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10232    // asks: an agent with a successor is asked once and handed over, and only
10233    // a seat with no successor spends `retries` nudges on the same agent. This
10234    // only guarantees the loop's own termination whatever those say.
10235    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10236    for round in 0..max_rounds {
10237        if pending.is_empty() {
10238            break;
10239        }
10240        let mut batch = Vec::with_capacity(pending.len());
10241        let mut renudged: Vec<&str> = Vec::new();
10242        for &i in &pending {
10243            let src = &originals[i];
10244            // A seat now held by another agent than the job named has no
10245            // session of its own: it gets the full-context prompt whenever
10246            // it is asked in full (the handover ask, a nudge it cannot
10247            // resume).
10248            let full: &str = match &src.handover {
10249                Some(h) if specs[i].id != src.spec.id => h,
10250                _ => &src.prompt,
10251            };
10252            // The prompt and the budget are one decision: a nudge restates
10253            // finished work, a re-sent prompt redoes it.
10254            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10255                (
10256                    full.to_owned(),
10257                    src.timeout,
10258                    format!("{}-{word}-{}", src.stem, specs[i].id),
10259                )
10260            } else if nudges[i] == 0 {
10261                (full.to_owned(), src.timeout, src.stem.clone())
10262            } else {
10263                renudged.push(src.seat.key.as_str());
10264                let why = done[i]
10265                    .as_ref()
10266                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10267                    .unwrap_or_else(|| "no parsable answer".to_owned());
10268                let nudge = prompt::nudge(&why);
10269                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10270                let prompt = if nudged {
10271                    nudge
10272                } else {
10273                    format!("{full}\n\n---\n\n{nudge}")
10274                };
10275                (
10276                    prompt,
10277                    retry_budget(src.timeout, nudged),
10278                    format!("{}-retry{}", src.stem, nudges[i]),
10279                )
10280            };
10281            batch.push(SeatJob {
10282                spec: specs[i].clone(),
10283                seat: seats[i].clone(),
10284                cwd: src.cwd.clone(),
10285                prompt,
10286                timeout,
10287                allow_write: src.allow_write,
10288                sessions: src.sessions,
10289                artifacts: src.artifacts.clone(),
10290                stem,
10291                handover: None,
10292            });
10293        }
10294
10295        if !renudged.is_empty() {
10296            state.event(
10297                ctx.node,
10298                format!("retry {round}: re-asking {}", renudged.join(", ")),
10299            );
10300        }
10301        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10302        let mut still = Vec::new();
10303        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10304            seats[i] = seat;
10305            // No agent was asked: nothing to hand over, nudge, remember or
10306            // charge. The caller sees `resource_block` and stops the node.
10307            if let AgentOutcome::ResourceBlocked(why) = &out {
10308                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10309                continue;
10310            }
10311            let class = FailClass::of(&out);
10312            // A dropped stream is nudged first (the conversation is still
10313            // there to pick up); only a seat whose nudges are spent hands over.
10314            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10315                && nudges[i] < retries
10316                && !roster.is_empty();
10317            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10318                let next = should_hand_over(prev[i].as_ref(), &cur)
10319                    .then(|| next_agent(i, &tried[i], &specs))
10320                    .flatten();
10321                if carry {
10322                    let h = state
10323                        .seat_history
10324                        .entry(originals[i].seat.key.clone())
10325                        .or_default();
10326                    h.failed.insert(specs[i].id.clone());
10327                    h.last_fail = Some(cur.clone());
10328                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10329                        h.last_ok = None;
10330                    }
10331                    // Saved before the next agent is asked, so a restart in
10332                    // between does not forget who failed.
10333                    if let Err(e) = state.save() {
10334                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10335                    }
10336                }
10337                if let Some(next) = next {
10338                    record_handover(
10339                        state,
10340                        ctx.node,
10341                        &originals[i].seat.key,
10342                        &specs[i].id,
10343                        &next.id,
10344                        &cur,
10345                        &fail_reason(&out),
10346                    );
10347                    tried[i].insert(next.id.clone());
10348                    prev[i] = Some(cur.clone());
10349                    seats[i] =
10350                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10351                    specs[i] = next;
10352                    fresh[i] = Some(cur.stem_word().to_owned());
10353                    nudges[i] = 0;
10354                    done[i] = Some(Err(anyhow::anyhow!(
10355                        "handed over after: {}",
10356                        fail_reason(&out)
10357                    )));
10358                    still.push(i);
10359                    continue;
10360                }
10361            }
10362            if carry
10363                && !nudge_first
10364                && let Some(cur) = class.clone()
10365            {
10366                // The chain ended here (no successor, or a repeated failure).
10367                let h = state
10368                    .seat_history
10369                    .entry(originals[i].seat.key.clone())
10370                    .or_default();
10371                h.failed.insert(specs[i].id.clone());
10372                h.last_fail = Some(cur);
10373            }
10374            let parsed = match out {
10375                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10376                AgentOutcome::Quota(o) => {
10377                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10378                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10379                }
10380                // Not a parseable answer: the nudge loop re-asks it, which is
10381                // exactly what a dropped stream needs. Just don't hand its raw
10382                // error JSON to `extract_json`.
10383                AgentOutcome::Dropped(o) => {
10384                    let why = o
10385                        .dropped
10386                        .as_ref()
10387                        .map(|d| d.why.as_str())
10388                        .unwrap_or("the CLI ended the stream without delivering its answer");
10389                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10390                }
10391                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10392                    Err(anyhow::anyhow!(e))
10393                }
10394            };
10395            let quota = class == Some(FailClass::Quota);
10396            let failed = parsed.is_err();
10397            done[i] = Some(parsed);
10398            if carry && !failed {
10399                let h = state
10400                    .seat_history
10401                    .entry(originals[i].seat.key.clone())
10402                    .or_default();
10403                h.failed.remove(&specs[i].id);
10404                h.last_ok = Some(specs[i].id.clone());
10405                h.last_fail = None;
10406            }
10407            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10408            // fail the same way; and never re-ask a seat that already parsed.
10409            // A failed agent that still has a successor is not re-asked
10410            // either: the handover was its remedy and has just been refused
10411            // (the chain stops on a repeated failure class). A seat with no
10412            // successor left (a single-agent roster, the roster's tail, or an
10413            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10414            let agent_failure = class.is_some()
10415                && !nudge_first
10416                && !roster.is_empty()
10417                && next_agent(i, &tried[i], &specs).is_some();
10418            if failed && !quota && !agent_failure && nudges[i] < retries {
10419                nudges[i] += 1;
10420                still.push(i);
10421            }
10422        }
10423        pending = still;
10424    }
10425
10426    // One loss per seat whose chain ended on a quota: the intermediate ones
10427    // were absorbed by a handover and are not losses.
10428    for (i, q) in last_quota.into_iter().enumerate() {
10429        if let Some(reset) = q {
10430            losses.push(QuotaLoss {
10431                seat: originals[i].seat.key.clone(),
10432                node: ctx.node.to_owned(),
10433                at: Timestamp::now(),
10434                reset,
10435            });
10436        }
10437    }
10438
10439    seats
10440        .into_iter()
10441        .zip(done)
10442        .zip(nudges)
10443        .map(|((seat, res), attempts)| {
10444            (
10445                seat,
10446                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10447                attempts,
10448            )
10449        })
10450        .collect()
10451}
10452
10453/// Did the wave end without asking any agent (see [`block_wave`])?
10454fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10455    results
10456        .iter()
10457        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10458}
10459
10460/// Settle a wave that never spawned anything: every job answers
10461/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10462/// else about the seat (session, history, quota) is touched.
10463fn block_wave(
10464    jobs: Vec<SeatJob>,
10465    state: &mut RunState,
10466    node: &str,
10467    attempt: usize,
10468    why: &str,
10469) -> Vec<(usize, SeatState, AgentOutcome)> {
10470    state.resource_wait = None;
10471    state
10472        .active
10473        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10474    if let Err(e) = state.save() {
10475        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10476    }
10477    jobs.into_iter()
10478        .enumerate()
10479        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10480        .collect()
10481}
10482
10483/// Acquire the shared build cache's lease, waiting out contention within
10484/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10485/// unbounded wait is never acceptable).
10486///
10487/// A first, non-blocking check happens before ever waiting; if it finds the
10488/// lease busy, that fact is logged as a `verify` event *and* flushed with
10489/// [`RunState::save`] immediately — not only once the wait finally succeeds
10490/// or gives up — so a `magi show` run by a different process while this one
10491/// is still waiting reads a `run.json` that says so, rather than whatever it
10492/// looked like before the wait started. The same applies to the terminal
10493/// failure: logged and saved before this returns `Err`, so a caller that
10494/// could not get the lease at all still leaves a legible record of why.
10495async fn acquire_cache_lease(
10496    state: &mut RunState,
10497    cache_dir: &Path,
10498    owner: &crate::cache::Owner,
10499    budget: Duration,
10500    node: &str,
10501    context: &str,
10502    cancel: &(dyn Fn() -> bool + Sync),
10503) -> Result<Option<crate::cache::Guard>> {
10504    let home = crate::run::home();
10505    let started = Instant::now();
10506    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10507        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10508        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10509        Err(e) => {
10510            state.event(
10511                "verify",
10512                format!("{context}: could not check the shared build cache: {e:#}"),
10513            );
10514            if let Err(e2) = state.save() {
10515                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10516            }
10517            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10518        }
10519    };
10520    state.event(
10521        "verify",
10522        format!(
10523            "{context}: waiting for the shared build cache at {} ({})",
10524            cache_dir.display(),
10525            busy.describe()
10526        ),
10527    );
10528    // Seats of this node are waiting, not running: say so, so the countdown a
10529    // reader sees only ever measures an agent that has been spawned.
10530    state.resource_wait = Some(crate::run::ResourceWait {
10531        node: context.to_owned(),
10532        kind: "shared build cache".to_owned(),
10533        holder: busy.describe(),
10534        since: Timestamp::now(),
10535        limit_secs: budget.as_secs(),
10536    });
10537    if let Err(e) = state.save() {
10538        tracing::warn!("could not persist a cache wait: {e:#}");
10539    }
10540    let remaining = budget.saturating_sub(started.elapsed());
10541    let waited = crate::cache::wait_for_unless(
10542        &home,
10543        cache_dir,
10544        owner,
10545        remaining,
10546        Duration::from_secs(5),
10547        cancel,
10548    )
10549    .await;
10550    state.resource_wait = None;
10551    let cause = match &waited {
10552        Ok(Some(_)) => None,
10553        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10554        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10555            Some(crate::run::ResourceBlockCause::Timeout {
10556                waited_secs: started.elapsed().as_secs(),
10557                limit_secs: budget.as_secs(),
10558            })
10559        }
10560        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10561    };
10562    if let Some(cause) = cause {
10563        let detail = match &waited {
10564            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10565            Err(error) => format!("{error:#}"),
10566        };
10567        state.resource_block = Some(crate::run::ResourceBlock {
10568            cause,
10569            node: node.to_owned(),
10570            kind: "shared build cache".into(),
10571            detail,
10572            at: Timestamp::now(),
10573        });
10574        state.event(
10575            "resource",
10576            state.resource_block.as_ref().unwrap().description(),
10577        );
10578    }
10579    if let Err(error) = state.save() {
10580        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10581    }
10582    waited
10583}
10584
10585/// The lease state could not be read at all (unreadable lease file, a
10586/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10587/// known to be contended, so a wave degrades to running without a shared cache
10588/// instead of stopping on a hold that [`crate::triage`] could never release.
10589#[derive(Debug)]
10590struct CacheCheckError(String);
10591
10592impl std::fmt::Display for CacheCheckError {
10593    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10594        f.write_str(&self.0)
10595    }
10596}
10597
10598impl std::error::Error for CacheCheckError {}
10599
10600/// Keep a specific acquisition outcome when the caller adds command evidence.
10601fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10602    state
10603        .resource_block
10604        .get_or_insert_with(|| crate::run::ResourceBlock {
10605            cause: crate::run::ResourceBlockCause::Other,
10606            node: node.to_owned(),
10607            kind: "shared build cache".into(),
10608            detail: format!("{error:#}"),
10609            at: Timestamp::now(),
10610        });
10611    if let Err(error) = state.save() {
10612        tracing::warn!("could not persist cache resource block: {error:#}");
10613    }
10614}
10615
10616/// Run `body` — a verify command batch — while holding the shared build
10617/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10618/// never interleave with another borrower's build against the same
10619/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10620/// timeout, or a human's own `magi review`. See the `cache` module doc for
10621/// why this matters more than Cargo's own per-target locking covers — two
10622/// *different* worktrees building the same package name/version into one
10623/// cache directory is a staleness bug, not a lock contention one.
10624///
10625/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10626/// agent waves do. Repository scheduling also has its own bounded,
10627/// cancellable resource-wait policy. Contention must not consume the
10628/// verification budget: `body` receives the full `budget` after both leases
10629/// are acquired and the cache is fresh. Without a shared cache, only
10630/// repository coordination is acquired before handing over the full budget.
10631///
10632/// A lease that cannot be acquired within the resource wait is reported as a single
10633/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10634/// verification — the same shape a spawn failure already takes in
10635/// [`run_commands`], so a caller need not special-case it.
10636#[allow(clippy::too_many_arguments)]
10637async fn with_cache_lease<'s, F, Fut>(
10638    state: &'s mut RunState,
10639    cache_dir: Option<&Path>,
10640    node: &str,
10641    seat: &str,
10642    worktree: &Path,
10643    head: &str,
10644    budget: Duration,
10645    context: &str,
10646    wait: &crate::repo_lock::WaitPolicy,
10647    body: F,
10648) -> Result<(Vec<CommandOutcome>, bool)>
10649where
10650    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10651    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10652{
10653    // Match editing waves: cache first, repository second. In particular,
10654    // pre_gate must never hold the repository while waiting for a wave's cache.
10655    let started = Instant::now();
10656    let home = crate::run::home();
10657    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10658    let guard = if let Some(cache_dir) = cache_dir {
10659        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10660        match acquire_cache_lease(
10661            state,
10662            cache_dir,
10663            &owner,
10664            resource_limit,
10665            node,
10666            context,
10667            &|| wait.cancelled(),
10668        )
10669        .await
10670        .and_then(|g| {
10671            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10672        }) {
10673            Ok(g) => Some(g),
10674            Err(e) => {
10675                record_cache_error(state, node, &e);
10676                return Ok((
10677                    vec![CommandOutcome {
10678                        command: "(waiting for the shared build cache)".into(),
10679                        code: None,
10680                        output_tail: e.to_string(),
10681                        duration_ms: started.elapsed().as_millis() as u64,
10682                        resource_blocked: true,
10683                    }],
10684                    false,
10685                ));
10686            }
10687        }
10688    } else {
10689        None
10690    };
10691    let _coordination = coordinate_operation(state, worktree, wait).await?;
10692    if let Some(cache_dir) = cache_dir {
10693        let identity = crate::cache::Identity::new(worktree, head);
10694        if let Err(e) = crate::cache::ensure_fresh(
10695            &home,
10696            cache_dir,
10697            &identity,
10698            &state.config.verify.cargo_manifests(),
10699        ) {
10700            record_cache_error(state, node, &e);
10701            state.event(
10702                "verify",
10703                format!(
10704                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10705                    worktree.display(),
10706                    short(head)
10707                ),
10708            );
10709            if let Some(guard) = guard {
10710                guard.release();
10711            }
10712            return Ok((
10713                vec![CommandOutcome {
10714                    command: "(confirming the shared build cache is fresh)".into(),
10715                    code: None,
10716                    output_tail: e.to_string(),
10717                    duration_ms: started.elapsed().as_millis() as u64,
10718                    resource_blocked: true,
10719                }],
10720                false,
10721            ));
10722        }
10723    }
10724    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10725    if !timed_out_pids.is_empty() {
10726        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10727    }
10728    // Drop repository coordination before releasing the cache, reversing the
10729    // acquisition order. A cancelled/error future also drops both guards.
10730    drop(_coordination);
10731    if let Some(guard) = guard {
10732        guard.release();
10733    }
10734    Ok((outcomes, retried))
10735}
10736
10737/// Poll `pids` — commands [`run_commands`] reports as still running when its
10738/// own timeout elapsed — until every one is confirmed gone, or
10739/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10740///
10741/// Real confirmation where confirmation is possible, not a substitute for
10742/// full process-tree observation: a grandchild the timed-out process spawned
10743/// and that survives independently of it is invisible to a pid check the
10744/// same way it always was, and continuing to observe and collect *that*
10745/// stays a different piece of work with its own owner. This only narrows a
10746/// fixed blind wait into an actual check of the pids this process does know
10747/// about.
10748async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10749    wait_for_pids_with(
10750        pids,
10751        crate::proc::pid_alive,
10752        LEASE_RELEASE_POLL,
10753        LEASE_RELEASE_MAX_WAIT,
10754    )
10755    .await;
10756}
10757
10758/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10759/// interval and ceiling supplied by the caller, so the polling *logic* -
10760/// returns as soon as every pid reports dead, gives up at the ceiling
10761/// otherwise - is testable on millisecond durations without asking the real
10762/// OS about a pid at all.
10763async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10764    pids: &[u32],
10765    alive: F,
10766    poll: Duration,
10767    max_wait: Duration,
10768) {
10769    let deadline = Instant::now() + max_wait;
10770    loop {
10771        if pids.iter().all(|&pid| !alive(pid)) {
10772            return;
10773        }
10774        if Instant::now() >= deadline {
10775            return;
10776        }
10777        tokio::time::sleep(poll).await;
10778    }
10779}
10780
10781/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10782/// admission that it could not even get a verify command to run, as opposed
10783/// to evidence the command actually produced? A caller that would otherwise
10784/// read a resource-blocked outcome as a red command must check this first:
10785/// see [`Runner::gate`], which retries rather than records `Blocked` when
10786/// this is true.
10787fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10788    outcomes.iter().any(|o| o.resource_blocked)
10789}
10790
10791/// What [`Runner::gate_fix_round`] decided.
10792pub(crate) enum GateFix {
10793    /// The tree changed and `verify.e2e` is still green: run the gate again.
10794    Retry,
10795    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10796    /// last failure stands and the run ends blocked.
10797    Stop,
10798    /// `verify.e2e` could not run after the fix (magi's own contention):
10799    /// decide nothing now, a later reentry retries.
10800    Defer,
10801}
10802
10803/// Is every red command in `outcomes` an ordinary failure the code could
10804/// explain: it ran, exited non-zero, and said something?
10805///
10806/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10807/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10808/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10809/// what the command is stays the gate's business.
10810fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10811    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10812    red.peek().is_some()
10813        && red.all(|o| {
10814            !o.resource_blocked
10815                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10816                && !o.output_tail.trim().is_empty()
10817        })
10818}
10819
10820/// Describe one verify command's outcome for the event log, distinguishing a
10821/// build/link failure — the toolchain never produced a binary to run — from
10822/// an actual test failure, since only the latter is a verdict on the patch.
10823fn e2e_outcome_label(o: &CommandOutcome) -> String {
10824    if o.ok() {
10825        return "pass".to_owned();
10826    }
10827    let reason = if o.build_failed() {
10828        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10829    } else {
10830        format!("FAIL ({:?})", o.code)
10831    };
10832    format!(
10833        "{reason}\n{}",
10834        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
10835    )
10836}
10837
10838/// Run `verify.e2e`, retrying once if the first attempt could not build or
10839/// link — a build/link failure is frequently a race against a shared
10840/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10841/// `verify` event per command, tagged with `context` (normally `"round N"`)
10842/// so the two call sites that need this — the ordinary per-round leg in
10843/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10844/// it will ever call a round green — read identically in the event log.
10845async fn run_e2e_with_retry(
10846    state: &mut RunState,
10847    shell: &[String],
10848    commands: &[String],
10849    worktree: &Path,
10850    timeout: Duration,
10851    context: &str,
10852) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10853    let (mut e2e, mut timed_out_pids) = run_commands(
10854        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10855    )
10856    .await;
10857    for o in &e2e {
10858        state.event(
10859            "verify",
10860            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10861        );
10862    }
10863    // A build/link failure is not a verdict on the patch — it is frequently a
10864    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10865    // one retry before letting a red like that decide the round.
10866    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10867    if verify_retried {
10868        state.event(
10869            "verify",
10870            format!(
10871                "{context}: verify could not build/link, not a test result — retrying once \
10872                 before concluding"
10873            ),
10874        );
10875        let retried = run_commands(
10876            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10877        )
10878        .await;
10879        e2e = retried.0;
10880        // Both attempts' timeouts matter, not just the last one: the first
10881        // attempt's descendants may still be alive alongside the retry's.
10882        timed_out_pids.extend(retried.1);
10883        for o in &e2e {
10884            state.event(
10885                "verify",
10886                format!(
10887                    "{context}: retry `{}` -> {}",
10888                    o.command,
10889                    e2e_outcome_label(o)
10890                ),
10891            );
10892        }
10893    }
10894    (e2e, verify_retried, timed_out_pids)
10895}
10896
10897/// Run configured shell commands in `cwd`, in order. The second element is
10898/// the pid of every command that hit `timeout` and was still running when
10899/// this stopped waiting on it (best-effort: `None` when the platform did not
10900/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10901/// that releases a shared resource afterward needs to know.
10902///
10903/// Records `task` into [`RunState::active`] at every command boundary
10904/// (`RunState::task_command`) and clears it once the whole list has run
10905/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10906/// for minutes with no seat and no output of its own to show for it (see
10907/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10908/// apart from "not yet run" without this), and this is the only place that
10909/// knows which command is running right now and how many are left. Three
10910/// saves per command — start, not per second — matching the same "only at a
10911/// boundary" rule [`wave`] already follows for seats.
10912#[allow(clippy::too_many_arguments)]
10913async fn run_commands(
10914    state: &mut RunState,
10915    node: &str,
10916    task: &str,
10917    attempt: usize,
10918    shell: &[String],
10919    commands: &[String],
10920    cwd: &Path,
10921    timeout: Duration,
10922) -> (Vec<CommandOutcome>, Vec<u32>) {
10923    if commands.is_empty() {
10924        // Nothing to mark as running and nothing to clear — an empty list
10925        // means "not configured", and touching `active` (or the disk) over
10926        // that would be a write for every round of a repo with no
10927        // `verify.e2e` / `verify.gate` commands at all.
10928        return (Vec::new(), Vec::new());
10929    }
10930    let mut out = Vec::new();
10931    let mut timed_out_pids = Vec::new();
10932    let total = commands.len();
10933    for (idx, command) in commands.iter().enumerate() {
10934        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10935        if let Err(e) = state.save() {
10936            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10937        }
10938        let started = Instant::now();
10939        let mut cmd = tokio::process::Command::new(&shell[0]);
10940        cmd.quiet();
10941        // Verify the checkout as a standalone caller. Worker ownership and
10942        // pinned executable selection belong to this driver, not to tests or
10943        // tools it runs. Preserve home/seat context and Running's containment.
10944        for key in crate::worker::INVOCATION_ENV {
10945            cmd.env_remove(key);
10946        }
10947        cmd.args(&shell[1..])
10948            .arg(command)
10949            .current_dir(cwd)
10950            .stdin(std::process::Stdio::null())
10951            .stdout(std::process::Stdio::piped())
10952            .stderr(std::process::Stdio::piped())
10953            .kill_on_drop(true);
10954        let mut outputs = Vec::new();
10955        let protected = (|| -> Result<()> {
10956            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10957                outputs.push(guard);
10958            }
10959            if let Some(cache) = state.config.cache_dir() {
10960                outputs.push(crate::retention::OutputLease::cache(
10961                    &crate::run::home(),
10962                    &cache,
10963                    false,
10964                )?);
10965            }
10966            Ok(())
10967        })();
10968        if let Err(e) = protected {
10969            out.push(CommandOutcome {
10970                command: command.clone(),
10971                code: None,
10972                output_tail: format!("output ownership unavailable: {e:#}"),
10973                duration_ms: 0,
10974                resource_blocked: true,
10975            });
10976            continue;
10977        }
10978        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10979        let (code, body) = match spawned {
10980            Ok(mut running) => {
10981                let pid = running.child().id();
10982                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10983                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10984                let mut paths = vec![cwd.to_owned()];
10985                if let Some(cache) = state.config.cache_dir() {
10986                    paths.push(cache);
10987                }
10988                let mut pressure = None;
10989                let status = tokio::select! {
10990                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10991                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10992                        pressure = Some(reason); None
10993                    }
10994                };
10995                let (code, failure) = match status {
10996                    Some(Ok(Ok(status))) => (
10997                        status.code(),
10998                        status
10999                            .code()
11000                            .is_none()
11001                            .then(|| format!("command terminated: {status}")),
11002                    ),
11003                    Some(Ok(Err(error))) => {
11004                        (None, Some(format!("failed waiting for command: {error}")))
11005                    }
11006                    Some(Err(_)) => {
11007                        // Leased cleanup confirms process-tree death itself.
11008                        // Unleased commands still need the caller's PID check
11009                        // after kill-on-drop, before releasing cache ownership.
11010                        if !running.coordinated()
11011                            && let Some(pid) = pid
11012                        {
11013                            timed_out_pids.push(pid);
11014                        }
11015                        (
11016                            None,
11017                            Some(format!("timed out after {}s", timeout.as_secs())),
11018                        )
11019                    }
11020                    None => (None, None),
11021                };
11022                running.finish().await;
11023                if let Some(reason) = pressure {
11024                    state.resource_block = Some(crate::run::ResourceBlock {
11025                        cause: crate::run::ResourceBlockCause::Other,
11026                        node: node.to_owned(),
11027                        kind: "disk pressure".to_owned(),
11028                        detail: reason.clone(),
11029                        at: Timestamp::now(),
11030                    });
11031                    out.push(CommandOutcome {
11032                        command: command.clone(),
11033                        code: None,
11034                        output_tail: reason,
11035                        duration_ms: started.elapsed().as_millis() as u64,
11036                        resource_blocked: true,
11037                    });
11038                    break;
11039                }
11040                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11041                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11042                if let Some(failure) = failure {
11043                    body.push('\n');
11044                    body.push_str(&failure);
11045                }
11046                (code, body)
11047            }
11048            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11049        };
11050        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11051        if !body.is_empty() {
11052            // The full text is kept whatever the summary dropped. A fresh
11053            // name per command so a resume or retry never overwrites an
11054            // earlier run's output. A failed write changes no verdict.
11055            match crate::run::write_verification_artifact(state, task, &body) {
11056                Ok(name) if summary != body => {
11057                    summary = format!("[full output: artifacts/{name}]\n{summary}");
11058                }
11059                Ok(_) => {}
11060                Err(e) => {
11061                    let warning = format!("could not save the full {task} output: {e:#}");
11062                    tracing::warn!("{warning}");
11063                    state.event(node, warning);
11064                    if summary != body {
11065                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11066                    }
11067                }
11068            }
11069        }
11070        out.push(CommandOutcome {
11071            command: command.clone(),
11072            code,
11073            output_tail: summary,
11074            duration_ms: started.elapsed().as_millis() as u64,
11075            resource_blocked: false,
11076        });
11077    }
11078    state.task_finished(task);
11079    if let Err(e) = state.save() {
11080        tracing::warn!("could not persist the end of {task}: {e:#}");
11081    }
11082    (out, timed_out_pids)
11083}
11084
11085/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
11086/// section (`report::run`) and in the `merge` event this node records — for
11087/// the operator to run by hand.
11088///
11089/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
11090/// branch whose ruleset forbids merge commits (GitHub's "must not contain
11091/// merge commits", or "require linear history") rejects the push a `--no-ff`
11092/// merge would produce, which is exactly the guidance this function replaces.
11093/// `message`'s first line becomes the squash commit's subject, matching the
11094/// note `report::run` prints alongside this command — see that function for
11095/// why an explicit subject is not optional there.
11096fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11097    let repo = repo.display();
11098    match style {
11099        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11100        MergeStyle::Squash => {
11101            // The subject sits inside double quotes, and a title an agent
11102            // wrote may carry the characters that break out of them.
11103            let subject = message
11104                .lines()
11105                .next()
11106                .unwrap_or(branch)
11107                .replace(['\\', '"', '$', '`'], "");
11108            format!(
11109                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11110            )
11111        }
11112        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11113    }
11114}
11115
11116/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
11117/// under the hood, rejects a `title` over 256 characters and the whole
11118/// command fails — no PR at all, for a run whose body was otherwise fine
11119/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
11120/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
11121/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
11122/// this bound. It is a margin, not a guarantee — a title packed
11123/// with multi-unit characters could still in principle land close to the
11124/// edge, but a real task title's occasional emoji or accented letter fits
11125/// comfortably inside it.
11126const PR_TITLE_MAX: usize = 240;
11127
11128/// A pull request title from the opening line of `text`, or `None` when that
11129/// line is not English (GitHub text is) or has no letters.
11130///
11131/// A line within `max` is kept as is. A longer one is cut at the end of its
11132/// first sentence when that falls inside `max`, else at a word boundary with a
11133/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
11134/// make every merely-truncated title look non-English). The language check
11135/// runs on the kept text before any mark is added, so only what GitHub will
11136/// show is judged: an English opening followed by non-ASCII far past the cut
11137/// still passes.
11138fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11139    let line = queue::first_line(text)?;
11140    let chars: Vec<char> = line.chars().collect();
11141    let (kept, mark) = if chars.len() <= max {
11142        (line.to_owned(), "")
11143    } else if let Some(end) = sentence_end(&chars, max) {
11144        (chars[..end].iter().collect::<String>(), "")
11145    } else {
11146        let room = max.saturating_sub(3);
11147        // Cut at the last space inside the room; when the char just past the
11148        // room is a space the room already ends on a word.
11149        let cut = if chars[room].is_whitespace() {
11150            room
11151        } else {
11152            chars[..room]
11153                .iter()
11154                .rposition(|c| c.is_whitespace())
11155                .unwrap_or(room)
11156        };
11157        let head: String = chars[..cut].iter().collect();
11158        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11159        (head.to_owned(), "...")
11160    };
11161    if kept.is_empty() {
11162        return None;
11163    }
11164    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11165        return None;
11166    }
11167    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11168        return None;
11169    }
11170    Some(format!("{kept}{mark}"))
11171}
11172
11173/// The char length of the first sentence of `chars` when it ends within `max`
11174/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
11175/// abbreviations so `e.g. foo` does not end a title early.
11176fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11177    const MIN: usize = 20;
11178    for i in MIN..max.min(chars.len()) {
11179        if !matches!(chars[i], '.' | '!' | '?') {
11180            continue;
11181        }
11182        let Some(&next) = chars.get(i + 1) else {
11183            continue;
11184        };
11185        if !next.is_whitespace() {
11186            continue;
11187        }
11188        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11189        if after.is_some_and(|c| c.is_ascii_lowercase()) {
11190            continue;
11191        }
11192        let word: String = chars[..i]
11193            .iter()
11194            .rev()
11195            .take_while(|c| !c.is_whitespace())
11196            .collect::<Vec<_>>()
11197            .into_iter()
11198            .rev()
11199            .collect();
11200        let word = word.to_ascii_lowercase();
11201        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11202            continue;
11203        }
11204        let end = chars[..i]
11205            .iter()
11206            .rposition(|c| !c.is_whitespace())
11207            .map_or(i, |p| p + 1);
11208        return Some(end);
11209    }
11210    None
11211}
11212
11213/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11214/// change: a title and a body describing what was *implemented*, not the task
11215/// that asked for it. A task reads as a request; a reader of the merged
11216/// history wants the change.
11217struct PrMessage {
11218    title: String,
11219    body: String,
11220}
11221
11222impl PrMessage {
11223    /// Title, blank line, body. The first line is the squash/merge commit
11224    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11225    /// has to stay one sensible line.
11226    fn commit_message(&self) -> String {
11227        format!("{}\n\n{}", self.title, self.body)
11228    }
11229}
11230
11231/// The text after a leading `TITLE:` (any case) on `line`.
11232fn title_marker(line: &str) -> Option<&str> {
11233    let line = line.trim();
11234    let head = line.get(..6)?;
11235    head.eq_ignore_ascii_case("title:")
11236        .then(|| line[6..].trim())
11237}
11238
11239/// The implementer's own one-line title: the `TITLE:` line the implement
11240/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11241/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11242/// source, and a title that says as much is refused here too.
11243fn summary_title(summary: &str) -> Option<String> {
11244    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11245    let raw = title_marker(first)?;
11246    if raw.is_empty() {
11247        return None;
11248    }
11249    let title = queue::title_from(raw, PR_TITLE_MAX);
11250    let lower = title.to_ascii_lowercase();
11251    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11252        return None;
11253    }
11254    Some(title)
11255}
11256
11257/// How `open_review`'s instruction begins; see [`landing_title`].
11258const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11259
11260/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11261pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11262
11263/// Does this run review work that already existed, rather than implement a
11264/// task? Runs recorded before `reviewed_commits` existed carry only the
11265/// candidate marker.
11266pub(crate) fn is_review_run(state: &RunState) -> bool {
11267    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11268}
11269
11270/// The title of a review-only run: the subject of the oldest commit under
11271/// review. Later commits are usually fixups, and `instruction` is the review
11272/// prompt, which says nothing about the change. GitHub text is English, so a
11273/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11274fn review_title(state: &RunState) -> Option<String> {
11275    english_subject(
11276        state.reviewed_commits.as_ref()?.first()?,
11277        state.config.graph.github_english(),
11278    )
11279}
11280
11281/// `raw` as a pull request title, or `None` when it is blank, not English
11282/// (GitHub text is), or one of magi's own candidate commit subjects.
11283fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11284    let raw = raw.trim();
11285    if raw.is_empty() {
11286        return None;
11287    }
11288    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11289        return None;
11290    }
11291    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11292        return None;
11293    }
11294    let title = queue::title_from(raw, PR_TITLE_MAX);
11295    let lower = title.to_ascii_lowercase();
11296    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11297        return None;
11298    }
11299    Some(title)
11300}
11301
11302/// What a review-only run's branch says about itself, read at the moment the
11303/// pull request is opened.
11304#[derive(Debug, Clone, PartialEq, Eq)]
11305struct BranchFacts {
11306    /// `(subject, body)` of each commit, oldest first.
11307    commits: Vec<(String, String)>,
11308    /// Trimmed `git diff --stat`.
11309    stat: String,
11310}
11311
11312/// Longest diff stat shown: this many file lines plus the summary line.
11313const STAT_FILE_LINES: usize = 25;
11314/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11315const COMMITS_MAX_CHARS: usize = 20_000;
11316
11317/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11318/// say or finds nothing, so the caller falls back to what the run recorded.
11319async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11320    let commits = git::commit_log(repo, base, branch).await.ok()?;
11321    if commits.is_empty() {
11322        return None;
11323    }
11324    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11325    let lines: Vec<&str> = stat.lines().collect();
11326    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11327        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11328        let more = format!(" ... {omitted} more file(s)");
11329        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11330        kept.push(&more);
11331        kept.push(lines[lines.len() - 1]);
11332        kept.join("\n")
11333    } else {
11334        lines.join("\n")
11335    };
11336    Some(BranchFacts { commits, stat })
11337}
11338
11339/// Defang what would break the surrounding markdown: a closing `</details>`
11340/// and a code fence.
11341fn markdown_safe(text: &str) -> String {
11342    text.replace("</details>", "&lt;/details&gt;")
11343        .replace("\x60\x60\x60", "~~~")
11344}
11345
11346fn neutral_title(state: &RunState, winner: char) -> String {
11347    format!(
11348        "chore: land candidate {} of run {}",
11349        winner.to_ascii_uppercase(),
11350        state.id
11351    )
11352}
11353
11354/// Reuse the graph's bounded fixer without starting another graph or driver.
11355/// Copy the working state back even on errors so spent rounds remain visible.
11356pub(crate) async fn repair_landing_gate(
11357    state: &mut RunState,
11358    winner: &Candidate,
11359    outcomes: &[CommandOutcome],
11360) -> Result<GateFix> {
11361    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11362        return Ok(GateFix::Stop);
11363    }
11364    let mut runner = Runner {
11365        roles: state.config.resolve_roles()?,
11366        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11367        state: state.clone(),
11368        pause: Pause::new(),
11369        interrupt: Pause::new(),
11370    };
11371    let result = runner.gate_fix_round(winner, outcomes).await;
11372    *state = runner.state;
11373    result
11374}
11375
11376/// Recheck a repair's e2e requirement across failures, resource deferrals and
11377/// process interruptions. A successful result only applies to its exact SHA.
11378pub(crate) async fn check_gate_repair_e2e(
11379    state: &mut RunState,
11380    winner: &Candidate,
11381    wait: &crate::repo_lock::WaitPolicy,
11382) -> Result<GateFix> {
11383    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11384        return Ok(GateFix::Retry);
11385    }
11386    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11387    if state
11388        .gate_repair_e2e
11389        .as_ref()
11390        .is_some_and(|r| r.head == head && r.passed)
11391    {
11392        return Ok(GateFix::Retry);
11393    }
11394    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11395        head: head.clone(),
11396        passed: false,
11397        outcomes: Vec::new(),
11398    });
11399    state.save()?;
11400    let commands = state.config.verify.e2e.clone();
11401    let outcomes = if commands.is_empty() {
11402        Vec::new()
11403    } else {
11404        let shell = state.config.shell();
11405        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11406        let cache_dir = state.config.cache_dir();
11407        let (outcomes, _) = with_cache_lease(
11408            state,
11409            cache_dir.as_deref(),
11410            "e2e",
11411            "e2e",
11412            &winner.worktree,
11413            &head,
11414            timeout,
11415            "gate repair e2e",
11416            wait,
11417            |state, budget| async move {
11418                run_e2e_with_retry(
11419                    state,
11420                    &shell,
11421                    &commands,
11422                    &winner.worktree,
11423                    budget,
11424                    "gate repair e2e",
11425                )
11426                .await
11427            },
11428        )
11429        .await?;
11430        outcomes
11431    };
11432    let deferred = verify_inconclusive(&outcomes);
11433    let passed = outcomes.iter().all(CommandOutcome::ok)
11434        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11435    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11436        head,
11437        passed,
11438        outcomes,
11439    });
11440    state.save()?;
11441    if deferred {
11442        Ok(GateFix::Defer)
11443    } else if !passed {
11444        state.event(
11445            "gate",
11446            "verify.e2e failed after gate repair; nothing published",
11447        );
11448        Ok(GateFix::Stop)
11449    } else {
11450        Ok(GateFix::Retry)
11451    }
11452}
11453
11454/// Run the same configured gate for a landing head and a fresh graph.
11455pub(crate) async fn run_landing_gate(
11456    state: &mut RunState,
11457    winner: &Candidate,
11458    wait: &crate::repo_lock::WaitPolicy,
11459) -> Result<Vec<CommandOutcome>> {
11460    let shell = state.config.shell();
11461    let gate_commands = state.config.verify.gate.clone();
11462    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11463    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11464        Vec::new()
11465    } else {
11466        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11467        let cache_dir = state.config.cache_dir();
11468        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11469        let (outcomes, _) = with_cache_lease(
11470            state,
11471            cache_dir.as_deref(),
11472            "gate",
11473            "gate",
11474            &winner.worktree,
11475            &head,
11476            timeout,
11477            "final gate",
11478            wait,
11479            |state, budget| {
11480                let shell = shell.clone();
11481                let gate_commands = gate_commands.clone();
11482                let winner = winner.clone();
11483                async move {
11484                    Runner::run_pre_gate(state, &winner).await;
11485                    let (outcomes, timed_out_pids) = run_commands(
11486                        state,
11487                        "gate",
11488                        "gate",
11489                        0,
11490                        &shell,
11491                        &gate_commands,
11492                        &winner.worktree,
11493                        budget,
11494                    )
11495                    .await;
11496                    (outcomes, false, timed_out_pids)
11497                }
11498            },
11499        )
11500        .await?;
11501        outcomes
11502    };
11503    if outcomes.is_empty() {
11504        // Nothing configured to check — distinct from every other
11505        // silence in this run's event log, since an empty `gate` alone
11506        // no longer says whether the gate ran at all (see
11507        // `RunState::gate_ran`'s own doc).
11508        state.event("gate", "no gate commands configured; not tested");
11509    }
11510    for o in &outcomes {
11511        state.event(
11512            "gate",
11513            format!(
11514                "`{}` -> {}",
11515                o.command,
11516                if o.ok() {
11517                    "pass".to_owned()
11518                } else {
11519                    format!(
11520                        "FAIL ({:?})\n{}",
11521                        o.code,
11522                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11523                    )
11524                }
11525            ),
11526        );
11527    }
11528    Ok(outcomes)
11529}
11530
11531/// The pull request title to hand to `land::merge_subject`. A review-only run
11532/// opened by an earlier build titled its pull request with the review prompt;
11533/// that title is dropped (empty, so the fallback applies) rather than landed.
11534/// Any other title, including an operator's rename, passes through untouched,
11535/// and so does every title of a run that implements a task.
11536pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11537    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11538        ""
11539    } else {
11540        pr_title
11541    }
11542}
11543
11544/// What the squash subject falls back to when the pull request title is empty
11545/// or candidate-shaped: for a review-only run the derived title, never the
11546/// review prompt held in `instruction`.
11547pub fn landing_subject_source(state: &RunState) -> String {
11548    if is_review_run(state) {
11549        let winner = state.candidates.first().map_or('A', |c| c.label);
11550        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11551    }
11552    state.instruction.clone()
11553}
11554
11555/// `summary` without its `TITLE:` line, which the pull request title already
11556/// carries.
11557fn summary_without_title(summary: &str) -> String {
11558    let mut lines = summary.trim().lines().peekable();
11559    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11560        lines.next();
11561    }
11562    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11563}
11564
11565/// The pull request title and body for the winning candidate.
11566///
11567/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11568/// the task's own opening line via [`queue::title_from`] when there is none.
11569/// `state.instruction` can open with blank lines (`task_text` only rejects a
11570/// body that is blank *entirely*), which `title_from` skips.
11571///
11572/// Publishing uses an independent description generated from the final diff
11573/// and structured evidence. Internal summaries and reports remain local.
11574/// Non-publishing previews may display the original task and candidate summary.
11575/// The footer retains the run and candidate provenance.
11576#[cfg(test)]
11577fn pr_message(state: &RunState, winner: char) -> PrMessage {
11578    pr_message_with(state, winner, None)
11579}
11580
11581/// [`pr_message`] with what the branch of a review-only run says about itself.
11582/// `facts` is ignored for a run that implements a task.
11583#[cfg(test)]
11584fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11585    let raw = pr_message_raw(
11586        state,
11587        winner,
11588        facts,
11589        crate::pr_outcome::FinalHead::Unreported,
11590        ChangeNote::NotAsked,
11591    );
11592    let id = crate::scrub::Identity::current();
11593    PrMessage {
11594        title: crate::scrub::scrub(&raw.title, &id),
11595        body: crate::scrub::scrub(&raw.body, &id),
11596    }
11597}
11598
11599/// What the public description may say about the change itself.
11600#[derive(Debug, Clone, Copy)]
11601enum ChangeNote<'a> {
11602    /// A local preview or non-publishing merge mode: no public generation needed.
11603    NotAsked,
11604    /// A summary written from the final diff.
11605    Summary(&'a crate::run::ChangeSummary),
11606    /// A summary was asked for and could not be produced; the text says why.
11607    Unavailable(&'a str),
11608}
11609
11610fn pr_message_raw(
11611    state: &RunState,
11612    winner: char,
11613    facts: Option<&BranchFacts>,
11614    head: crate::pr_outcome::FinalHead<'_>,
11615    change: ChangeNote<'_>,
11616) -> PrMessage {
11617    if !matches!(change, ChangeNote::NotAsked) {
11618        let (title, mut body) = match change {
11619            ChangeNote::Summary(c) => (
11620                c.title.clone(),
11621                format!("## Summary\n\n{}\n\n", c.body.trim()),
11622            ),
11623            ChangeNote::Unavailable(why) => (
11624                neutral_title(state, winner),
11625                format!(
11626                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11627                ),
11628            ),
11629            ChangeNote::NotAsked => unreachable!(),
11630        };
11631        body.push_str(&public_pr_evidence(state, head));
11632        body.push_str(&format!(
11633            "\n---\nmagi:run/{} magi:candidate-{}\n",
11634            state.id,
11635            winner.to_ascii_lowercase()
11636        ));
11637        return PrMessage { title, body };
11638    }
11639    pr_preview(state, winner, facts, head)
11640}
11641
11642/// Append public facts from the run records after every prose generation,
11643/// including posting-gate rewrites. The agent cannot drop this provenance.
11644fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11645    let mut body = crate::pr_outcome::review_rounds(state);
11646    body.push_str(&crate::pr_outcome::final_verification(state, head));
11647    let open = state.last_round_findings();
11648    if !open.is_empty() {
11649        body.push_str("## Open review findings\n\n");
11650        for f in open {
11651            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11652        }
11653        body.push('\n');
11654    }
11655    if let Some(c) = state.change_summary.as_ref()
11656        && head == crate::pr_outcome::FinalHead::At(&c.head)
11657    {
11658        body.push_str(&format!(
11659            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11660            c.base, c.head
11661        ));
11662    }
11663    body
11664}
11665
11666/// Local-only preview of the operator's task and candidate summary.
11667fn pr_preview(
11668    state: &RunState,
11669    winner: char,
11670    facts: Option<&BranchFacts>,
11671    head: crate::pr_outcome::FinalHead<'_>,
11672) -> PrMessage {
11673    let summary = state
11674        .candidates
11675        .iter()
11676        .find(|c| c.label == winner)
11677        .map(|c| c.summary.as_str())
11678        .unwrap_or_default();
11679    // The fallback is the operator's own words and may not be English; GitHub
11680    // text always is, so a non-English task gets a neutral title instead.
11681    let review = is_review_run(state);
11682    let title = if review {
11683        facts
11684            .and_then(|f| {
11685                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11686            })
11687            .or_else(|| review_title(state))
11688            .or_else(|| {
11689                state
11690                    .candidates
11691                    .iter()
11692                    .find(|c| c.label == winner)
11693                    .filter(|c| !c.branch.starts_with("magi/"))
11694                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11695            })
11696            .unwrap_or_else(|| neutral_title(state, winner))
11697    } else {
11698        summary_title(summary).unwrap_or_else(|| {
11699            english_title(
11700                &state.instruction,
11701                PR_TITLE_MAX,
11702                state.config.graph.github_english(),
11703            )
11704            .unwrap_or_else(|| neutral_title(state, winner))
11705        })
11706    };
11707
11708    let mut body = String::new();
11709    let what = summary_without_title(summary);
11710    if !what.is_empty() {
11711        body.push_str("## Summary\n\n");
11712        body.push_str(&what);
11713        body.push_str("\n\n");
11714    }
11715
11716    // Review history uses structured facts across all rounds, never reports.
11717    body.push_str(&crate::pr_outcome::review_rounds(state));
11718    body.push_str(&crate::pr_outcome::final_verification(state, head));
11719    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11720
11721    let open = state.open_findings();
11722    if !open.is_empty() {
11723        body.push_str("## Open review findings\n\n");
11724        for f in &open {
11725            body.push_str(&format!(
11726                "- `{}` [{:?}] {}\n",
11727                f.id, f.severity, "Context retained in local review records."
11728            ));
11729        }
11730        body.push('\n');
11731    }
11732
11733    if let Some(fix) = fix
11734        && !fix.rejected.is_empty()
11735    {
11736        body.push_str("## Declined by the fixer\n\n");
11737        for r in &fix.rejected {
11738            body.push_str(&format!(
11739                "- `{}`: {}\n",
11740                r.id, "Reason retained in local review records."
11741            ));
11742        }
11743        body.push('\n');
11744    }
11745
11746    if review {
11747        // The review prompt is not the task; list what the branch carries.
11748        body.push_str("## Commits under review\n\n");
11749        if let Some(facts) = facts {
11750            let mut left = COMMITS_MAX_CHARS;
11751            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11752                let mut entry = format!("- {}\n", markdown_safe(subject));
11753                for l in markdown_safe(text).lines() {
11754                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11755                }
11756                if left == 0 {
11757                    body.push_str(&format!(
11758                        "- ... {} more commit(s)\n",
11759                        facts.commits.len() - i
11760                    ));
11761                    break;
11762                }
11763                if entry.len() > left {
11764                    // Even the first commit is cut: one huge body must not
11765                    // push the whole description past GitHub's limit.
11766                    let mut end = left;
11767                    while !entry.is_char_boundary(end) {
11768                        end -= 1;
11769                    }
11770                    entry.truncate(end);
11771                    entry.push_str("\n  ... (truncated)\n");
11772                    left = 0;
11773                } else {
11774                    left -= entry.len();
11775                }
11776                body.push_str(&entry);
11777            }
11778            if !facts.stat.trim().is_empty() {
11779                body.push_str(&format!(
11780                    "\n## Diff stat\n\n```\n{}\n```\n",
11781                    markdown_safe(facts.stat.trim())
11782                ));
11783            }
11784        } else {
11785            match &state.reviewed_commits {
11786                Some(subjects) => {
11787                    for s in subjects {
11788                        body.push_str(&format!("- {}\n", s.trim()));
11789                    }
11790                }
11791                None => {
11792                    // An older run kept only the prompt, with the commit list
11793                    // after its first paragraph.
11794                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11795                    body.push_str(rest.trim());
11796                    body.push('\n');
11797                }
11798            }
11799        }
11800    } else {
11801        let task = state.instruction.trim();
11802        let task = if task.is_empty() {
11803            "(empty task)"
11804        } else {
11805            task
11806        };
11807        body.push_str(&format!(
11808            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11809            task.replace("</details>", "&lt;/details&gt;")
11810        ));
11811    }
11812
11813    body.push_str(&format!(
11814        "\n---\nmagi:run/{} magi:candidate-{}\n",
11815        state.id,
11816        winner.to_ascii_lowercase()
11817    ));
11818
11819    PrMessage { title, body }
11820}
11821
11822/// Source imports have already been prepared from structured inputs. Legacy
11823/// inferred references must never instruct a seat to reconstruct those imports.
11824fn seeded_instruction(state: &RunState) -> String {
11825    let mut inputs = Vec::new();
11826    if let Some(start) = &state.sources.start_from {
11827        inputs.push(format!(
11828            "Start from `{}` at `{}` (including ancestry).",
11829            start.token, start.sha
11830        ));
11831    }
11832    for pick in &state.sources.pick_commits {
11833        inputs.push(format!(
11834            "Pick `{}` at `{}` (single commit).",
11835            pick.token, pick.sha
11836        ));
11837    }
11838    if inputs.is_empty() {
11839        state.instruction.clone()
11840    } else {
11841        format!(
11842            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11843            state.instruction,
11844            inputs.join("\n")
11845        )
11846    }
11847}
11848
11849/// Does the winner have no commits ahead of the base it would land on?
11850/// Any failure to find out reads as "not empty": the merge then behaves as it
11851/// always did rather than refusing on a guess.
11852async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11853    let base = &state.base_branch;
11854    let mut against = base.clone();
11855    if mode == MergeMode::Pr {
11856        // A pull request lands on the remote's base, never the local branch
11857        // of the same name: if that cannot be read, "not empty" is the safe
11858        // answer.
11859        let remote = &state.config.merge.remote;
11860        let tracking = format!("{remote}/{base}");
11861        let fetched = git::fetch(repo, remote, base).await;
11862        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11863            against = tracking;
11864        } else {
11865            return false;
11866        }
11867    }
11868    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11869}
11870
11871/// Why nothing was opened for an empty winner, with what the task's own
11872/// references resolved to.
11873fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11874    let mut detail = format!(
11875        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11876         nothing to open a pull request for"
11877    );
11878    detail.push_str(
11879        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11880    );
11881    detail
11882}
11883
11884/// What the `Pr` merge does once it knows whether the branch already has an
11885/// open pull request.
11886#[derive(Debug, PartialEq, Eq)]
11887enum PrPlan {
11888    Create,
11889    Adopt { url: String, title: String },
11890    Stop(String),
11891}
11892
11893/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11894/// a failed lookup -> stop with the real reason. Never guesses.
11895fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11896    match found {
11897        Ok(land::OpenPr::None) => PrPlan::Create,
11898        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11899        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11900            "several open pull requests exist for this branch, not picking one: {}",
11901            urls.join(" ")
11902        )),
11903        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11904    }
11905}
11906
11907/// `gh pr create`, returning the PR url.
11908pub(crate) async fn gh_pr_create(
11909    cwd: &Path,
11910    base: &str,
11911    head: &str,
11912    title: &str,
11913    body: &str,
11914) -> Result<String> {
11915    let out = tokio::process::Command::new("gh")
11916        .args([
11917            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11918        ])
11919        .current_dir(cwd)
11920        .quiet()
11921        .stdin(std::process::Stdio::null())
11922        .output()
11923        .await
11924        .context("spawn gh")?;
11925    if out.status.success() {
11926        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11927    } else {
11928        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11929    }
11930}
11931
11932/// Never force deletion of uncommitted work, including unknown legacy trees.
11933async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11934    match std::fs::symlink_metadata(path) {
11935        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11936            // A previous cleanup may have removed the tree before a branch
11937            // deletion or state save failed. Unregister it if still present.
11938            git::worktree_remove_clean(repo, path).await?;
11939            return Ok(());
11940        }
11941        Err(e) => return Err(e.into()),
11942        Ok(_) => {}
11943    }
11944    if !git::is_clean(path).await? {
11945        bail!(
11946            "worktree has uncommitted work; refusing removal: {}",
11947            path.display()
11948        );
11949    }
11950    if !git::worktree_remove_clean(repo, path).await? {
11951        bail!("worktree removal refused: {}", path.display());
11952    }
11953    Ok(())
11954}
11955
11956/// Tear a run's worktrees and branches down.
11957///
11958/// `home` is where the updated `run.json` is saved (via
11959/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11960/// a housekeeping pass already has its own honest `home` handed to it, and
11961/// falling through to the global here would write back through whichever
11962/// directory some other process or test pinned into that `OnceLock` first,
11963/// not the one the caller actually resolved its `runs` and `state` from.
11964pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11965    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11966        bail!("refusing fold: {reason}");
11967    }
11968    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11969        bail!("candidate preservation incomplete; refusing cleanup");
11970    }
11971    // Serialize ordinary folding with the same starts/resumes as target-only
11972    // eviction. Otherwise a second janitor could remove the whole worktree
11973    // around a live output borrower even though target eviction was fenced.
11974    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11975    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11976    let _driver = crate::driver::maintenance(home, &state.id)?;
11977    let mut output_guards = Vec::new();
11978    for entry in std::fs::read_dir(state.worktree_root())
11979        .into_iter()
11980        .flatten()
11981        .flatten()
11982    {
11983        if entry.path().join(".git").is_file() {
11984            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11985        }
11986    }
11987    let repo = state.repo.clone();
11988    // CLI, Web and automatic cleanup all enter here, independently of
11989    // Runner::fold_losers. Protect deletion and shared config release too.
11990    let wait = repository_wait(state, None);
11991    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11992        Ok(lease) => lease,
11993        Err(e) => {
11994            state.save_under(home)?;
11995            return Err(e);
11996        }
11997    };
11998    // Confirmed merges dispose of their owned sources after durable evidence
11999    // capture. Preflight every source before removing anything.
12000    let _source_outputs = if state.status == RunStatus::Merged
12001        && state.merge.as_ref().is_some_and(|merge| merge.ok)
12002    {
12003        verify_merged_fold_sources(state, drop_winner, home).await?
12004    } else {
12005        Vec::new()
12006    };
12007    let root = state.worktree_root();
12008    let winner = state.tally.as_ref().map(|t| t.winner);
12009    let mut removed = Vec::new();
12010
12011    for i in 0..state.candidates.len() {
12012        let c = state.candidates[i].clone();
12013        let is_winner = Some(c.label) == winner;
12014        if is_winner && !drop_winner {
12015            continue;
12016        }
12017        let existed = c.worktree.exists();
12018        // A vanished directory can still have a Git registration that keeps
12019        // its branch checked out. Remove that registration before the branch.
12020        remove_preserved_worktree(&repo, &c.worktree).await?;
12021        if existed {
12022            removed.push(c.worktree.to_string_lossy().into_owned());
12023        }
12024        // A branch handed to a later run (and its pull request) is not this
12025        // run's to delete.
12026        let handed_over = state.released_branches.contains(&c.branch);
12027        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12028            if !git::branch_delete(&repo, &c.branch).await? {
12029                bail!("candidate branch removal refused");
12030            }
12031            removed.push(c.branch.clone());
12032        }
12033        state.candidates[i].folded = true;
12034    }
12035
12036    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12037        let path = name.path();
12038        let keep = !drop_winner
12039            && winner.is_some_and(|w| {
12040                path.file_name()
12041                    .is_some_and(|n| n == format!("cand-{w}").as_str())
12042            });
12043        if keep {
12044            continue;
12045        }
12046        remove_preserved_worktree(&repo, &path).await?;
12047        removed.push(path.to_string_lossy().into_owned());
12048    }
12049
12050    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
12051    // judge worktrees, so once the loop above has cleared all of them out,
12052    // the parent is a bare directory nobody else was ever going to remove -
12053    // git only ever managed what was inside it. Left alone, one of these
12054    // accumulates per fully-folded run; the operator's own machine had 74.
12055    // `remove_if_empty` re-checks rather than assuming: a run whose winner
12056    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
12057    // so does anything a run never claimed that happens to share the bay.
12058    remove_if_empty(&root);
12059
12060    if state.enabled_worktree_config && drop_winner {
12061        // A release, not a raw disable: some sibling run in this repository
12062        // may still hold its own reference (see `git::acquire_worktree_config`),
12063        // and only the last release actually turns the setting back off.
12064        git::release_worktree_config(&repo).await.ok();
12065        state.enabled_worktree_config = false;
12066    }
12067    state.save_under(home)?;
12068    Ok(removed)
12069}
12070
12071async fn verify_merged_fold_sources(
12072    state: &RunState,
12073    drop_winner: bool,
12074    home: &Path,
12075) -> Result<Vec<std::fs::File>> {
12076    let winner = state.tally.as_ref().map(|t| t.winner);
12077    let mut heads = BTreeSet::new();
12078    for candidate in &state.candidates {
12079        if Some(candidate.label) == winner && !drop_winner {
12080            continue;
12081        }
12082        anyhow::ensure!(
12083            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12084            "candidate recovery source is outside the inventoried run root: {}",
12085            candidate.worktree.display()
12086        );
12087        match std::fs::symlink_metadata(&candidate.worktree) {
12088            Ok(_) => {}
12089            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12090            Err(error) => return Err(error.into()),
12091        }
12092        if !state.released_branches.contains(&candidate.branch)
12093            && git::branch_exists(&state.repo, &candidate.branch).await?
12094        {
12095            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12096        }
12097    }
12098    let mut cargo_guards = Vec::new();
12099    let entries = match std::fs::read_dir(state.worktree_root()) {
12100        Ok(entries) => Some(entries),
12101        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12102        Err(error) => return Err(error.into()),
12103    };
12104    for entry in entries.into_iter().flatten() {
12105        let path = entry?.path();
12106        let metadata = std::fs::symlink_metadata(&path)?;
12107        anyhow::ensure!(
12108            metadata.is_dir() && !metadata.file_type().is_symlink(),
12109            "non-directory or symlink recovery source retained: {}",
12110            path.display()
12111        );
12112        if !drop_winner
12113            && winner.is_some_and(|w| {
12114                path.file_name()
12115                    .is_some_and(|name| name == format!("cand-{w}").as_str())
12116            })
12117        {
12118            continue;
12119        }
12120        anyhow::ensure!(
12121            path.join(".git").is_file(),
12122            "unregistered recovery source retained: {}",
12123            path.display()
12124        );
12125        anyhow::ensure!(
12126            git::git(
12127                &path,
12128                &[
12129                    "status",
12130                    "--porcelain",
12131                    "--untracked-files=all",
12132                    "--ignored",
12133                    "--",
12134                    ".",
12135                    ":(exclude)target"
12136                ]
12137            )
12138            .await?
12139            .is_empty(),
12140            "source changes or ignored recovery files retained: {}",
12141            path.display()
12142        );
12143        match std::fs::symlink_metadata(path.join("target")) {
12144            Ok(_) => {
12145                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12146            }
12147            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12148            Err(error) => return Err(error.into()),
12149        }
12150        for checkpoint in [
12151            "rebase-merge",
12152            "rebase-apply",
12153            "sequencer",
12154            "CHERRY_PICK_HEAD",
12155            "MERGE_HEAD",
12156            "REBASE_HEAD",
12157            "REVERT_HEAD",
12158            "BISECT_LOG",
12159            "AUTO_MERGE",
12160            "MERGE_MSG",
12161        ] {
12162            let location = git::git(
12163                &path,
12164                &[
12165                    "rev-parse",
12166                    "--path-format=absolute",
12167                    "--git-path",
12168                    checkpoint,
12169                ],
12170            )
12171            .await?;
12172            anyhow::ensure!(
12173                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12174                if e.kind() == std::io::ErrorKind::NotFound),
12175                "Git recovery checkpoint retained: {checkpoint}"
12176            );
12177        }
12178        heads.insert(git::rev_parse(&path, "HEAD").await?);
12179    }
12180    crate::fold_evidence::preserve(state, home, &heads).await?;
12181
12182    Ok(cargo_guards)
12183}
12184
12185/// Remove `dir` if it exists and has nothing in it.
12186///
12187/// Best-effort and silent by design: a directory that is not empty (a run
12188/// whose winner is still parked there, a stray file some other process left)
12189/// is exactly the case this must refuse, and a directory that is already gone
12190/// is not a failure worth reporting either. `std::fs::remove_dir` itself
12191/// already refuses a non-empty directory, so the emptiness check below is
12192/// belt, not suspenders - it is what keeps this from ever attempting the
12193/// removal in the case that matters, rather than trusting `remove_dir`'s
12194/// error path to have no side effects if it ever changed.
12195fn remove_if_empty(dir: &Path) {
12196    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12197        std::fs::remove_dir(dir).ok();
12198    }
12199}
12200
12201/// Severity of the worst open finding in the last review round, for reporting.
12202pub fn worst_open(state: &RunState) -> Option<Severity> {
12203    state
12204        .reviews
12205        .last()?
12206        .reviews
12207        .iter()
12208        .flat_map(|r| r.findings.iter())
12209        .map(|f| f.severity)
12210        .max()
12211}
12212
12213/// `[worktree] setup` hides its products per worktree, which needs
12214/// `extensions.worktreeConfig`. Taken once per run and released with the run's
12215/// worktrees, like the hook's; every place that runs setup calls this first.
12216async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12217    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12218        git::acquire_worktree_config(repo).await?;
12219        state.enabled_worktree_config = true;
12220    }
12221    Ok(())
12222}
12223
12224#[cfg(test)]
12225mod tests {
12226    #[tokio::test]
12227    async fn launcher_base_fetch_requires_remote_success() {
12228        let tmp = tempfile::tempdir().unwrap();
12229        let repo = tmp.path().join("repo");
12230        let remote = tmp.path().join("remote");
12231        std::fs::create_dir(&repo).unwrap();
12232        std::fs::create_dir(&remote).unwrap();
12233        git::git(&remote, &["init", "--bare", "-b", "main"])
12234            .await
12235            .unwrap();
12236        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12237        git::git(
12238            &repo,
12239            &[
12240                "-c",
12241                "user.name=Fixture",
12242                "-c",
12243                "user.email=fixture@example.invalid",
12244                "commit",
12245                "--allow-empty",
12246                "-m",
12247                "fixture",
12248            ],
12249        )
12250        .await
12251        .unwrap();
12252        git::git(
12253            &repo,
12254            &["remote", "add", "origin", remote.to_str().unwrap()],
12255        )
12256        .await
12257        .unwrap();
12258        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12259        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12260        assert_eq!(
12261            resolve_base(&repo, "main", "origin", None)
12262                .await
12263                .unwrap()
12264                .commit,
12265            head
12266        );
12267        #[cfg(unix)]
12268        {
12269            let policy: crate::config::Coordination = toml::from_str(
12270                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12271            ).unwrap();
12272            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12273            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12274            let held = crate::repo_lock::SeatLease::operation_with(
12275                &repo,
12276                Some(&policy),
12277                &crate::repo_lock::WaitPolicy::default(),
12278            )
12279            .await
12280            .unwrap();
12281            assert!(held.is_some());
12282            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12283            tokio::pin!(fetch);
12284            assert!(
12285                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12286                    .await
12287                    .is_err()
12288            );
12289            assert!(
12290                !repo.join(".git/FETCH_HEAD").exists(),
12291                "no fetch before admission"
12292            );
12293            drop(held);
12294            let fetched = fetch.await.unwrap();
12295            assert_eq!(fetched.commit, head);
12296            assert_eq!(fetched.remote, "origin");
12297            assert_eq!(fetched.branch, "main");
12298        }
12299        // A surviving tracking ref is not successful fetch evidence.
12300        git::git(
12301            &repo,
12302            &[
12303                "remote",
12304                "set-url",
12305                "origin",
12306                tmp.path().join("missing").to_str().unwrap(),
12307            ],
12308        )
12309        .await
12310        .unwrap();
12311        let error = resolve_base(&repo, "main", "origin", None)
12312            .await
12313            .unwrap_err()
12314            .to_string();
12315        assert!(error.contains("cannot read origin/main"));
12316        assert!(error.contains("refusing to branch off"));
12317    }
12318
12319    #[test]
12320    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12321        let leaked = vec!["chore(deps): update a crate".to_owned()];
12322        let own = "fix(daemon): apply a chosen action";
12323        assert!(should_retitle("", own, &leaked));
12324        assert!(should_retitle(
12325            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12326            own,
12327            &leaked
12328        ));
12329        assert!(should_retitle(
12330            "chore: land candidate A of run 1",
12331            own,
12332            &leaked
12333        ));
12334        assert!(should_retitle(
12335            "magi: candidate A (uncommitted work)",
12336            own,
12337            &leaked
12338        ));
12339        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12340        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12341        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12342    }
12343
12344    #[test]
12345    fn pr_merge_plan_creates_adopts_or_stops() {
12346        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12347        assert_eq!(
12348            pr_merge_plan(Ok(land::OpenPr::One {
12349                url: "u".into(),
12350                title: "t".into()
12351            })),
12352            PrPlan::Adopt {
12353                url: "u".into(),
12354                title: "t".into()
12355            }
12356        );
12357        let PrPlan::Stop(many) =
12358            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12359        else {
12360            panic!("many must stop");
12361        };
12362        assert!(many.contains('a') && many.contains('b'));
12363        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12364            panic!("a failed lookup must stop");
12365        };
12366        assert!(err.contains("bad token"));
12367    }
12368
12369    use super::*;
12370    use crate::run::GateStatus;
12371    use std::collections::BTreeMap;
12372    use std::time::Duration;
12373
12374    fn conductor() -> AgentSpec {
12375        AgentSpec {
12376            id: "conductor".to_owned(),
12377            kind: crate::config::AgentKind::Command,
12378            model: None,
12379            command: vec!["true".to_owned()],
12380            extra_args: Vec::new(),
12381            env: BTreeMap::new(),
12382            prompt_delivery: None,
12383            max_turn_secs: None,
12384        }
12385    }
12386
12387    fn spec(id: &str) -> AgentSpec {
12388        AgentSpec {
12389            id: id.to_owned(),
12390            kind: crate::config::AgentKind::Command,
12391            model: None,
12392            command: vec!["true".to_owned()],
12393            extra_args: Vec::new(),
12394            env: BTreeMap::new(),
12395            prompt_delivery: None,
12396            max_turn_secs: None,
12397        }
12398    }
12399
12400    fn ids(xs: &[&str]) -> BTreeSet<String> {
12401        xs.iter().map(|s| (*s).to_owned()).collect()
12402    }
12403
12404    #[test]
12405    fn pick_successor_skips_an_agent_another_seat_holds() {
12406        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12407        let roster = [spec("a"), spec("b"), spec("c")];
12408        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12409        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12410    }
12411
12412    #[test]
12413    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12414        // The other seat started on c but was handed to d; c is free again.
12415        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12416        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12417        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12418        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12419        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12420    }
12421
12422    #[test]
12423    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12424        let roster = [spec("a"), spec("b")];
12425        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12426        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12427        let carried = ids(&["b"]);
12428        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12429        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12430    }
12431
12432    #[test]
12433    fn pick_successor_returns_none_without_any_untried_successor() {
12434        let roster = [spec("a"), spec("b")];
12435        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12436        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12437        let carried = ids(&["a"]);
12438        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12439    }
12440
12441    #[test]
12442    fn pick_successor_rescue_avoids_another_seats_occupant() {
12443        // b is a carried failure and free; a is held by the other seat.
12444        let roster = [spec("a"), spec("b"), spec("c")];
12445        let carried = ids(&["b", "c"]);
12446        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12447        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12448    }
12449
12450    #[test]
12451    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12452        let roster = [spec("a"), spec("b"), spec("c")];
12453        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12454        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12455    }
12456
12457    #[test]
12458    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12459        let roster = [spec("a"), spec("b"), spec("c")];
12460        let failed = ids(&["a", "b", "c"]);
12461        // Rescue looks at the whole roster, once per id, then runs out.
12462        let mut tried = ids(&["b"]);
12463        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12464        assert_eq!(first.id, "a");
12465        tried.insert(first.id.clone());
12466        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12467        assert_eq!(second.id, "c");
12468        tried.insert(second.id.clone());
12469        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12470    }
12471
12472    #[test]
12473    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12474        let roster = [spec("a"), spec("b")];
12475        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12476        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12477    }
12478
12479    #[test]
12480    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12481        let roster = [spec("a"), spec("b"), spec("c")];
12482        let h = SeatHistory {
12483            failed: ids(&["a"]),
12484            last_ok: Some("b".to_owned()),
12485            last_fail: None,
12486        };
12487        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12488        // A last answerer that left the roster, or later failed, is ignored.
12489        let gone = SeatHistory {
12490            last_ok: Some("zzz".to_owned()),
12491            ..h.clone()
12492        };
12493        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12494        let failed = SeatHistory {
12495            failed: ids(&["a", "b"]),
12496            last_ok: Some("b".to_owned()),
12497            last_fail: None,
12498        };
12499        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12500    }
12501
12502    #[test]
12503    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12504        let first = SeatState::new("review-1", "alpha", 7);
12505        let next = handover_seat("review-1", "gamma", 7);
12506        assert_ne!(first.claude_session, next.claude_session);
12507    }
12508
12509    #[test]
12510    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12511        let mut state = state_with_summary("x", "y");
12512        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12513        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12514        assert_ne!(a.claude_session, b.claude_session);
12515    }
12516
12517    #[test]
12518    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12519        let roster = [spec("a"), spec("b")];
12520        let h = SeatHistory {
12521            failed: ids(&["a", "b"]),
12522            ..SeatHistory::default()
12523        };
12524        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12525        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12526        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12527    }
12528
12529    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12530    // fallback loop depends on to terminate: it must walk forward from the
12531    // seat's own position, never restart at the front of the roster, and it
12532    // must never hand back an id already tried, however many times that id
12533    // happens to appear.
12534
12535    #[test]
12536    fn failure_signature_ignores_numbers_and_paths() {
12537        assert_eq!(
12538            failure_signature("exited with Some(2) and no usable output"),
12539            failure_signature("exited with Some(137) and no usable output")
12540        );
12541        assert_eq!(
12542            failure_signature("process exited with no usable output (exit code 2)"),
12543            failure_signature("process exited with no usable output (exit code 137)")
12544        );
12545        for (old, new) in [
12546            (
12547                "exited with Some(2) and no usable output",
12548                "process exited with no usable output (exit code 2)",
12549            ),
12550            (
12551                "exited with None and no usable output",
12552                "process exited with no usable output (exit code unavailable)",
12553            ),
12554        ] {
12555            let previous = FailClass::Other(failure_signature(old));
12556            let current = FailClass::Other(failure_signature(new));
12557            assert_eq!(previous, current, "resumption retains the failure class");
12558            assert!(!should_hand_over(Some(&previous), &current));
12559        }
12560        assert_eq!(
12561            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12562            failure_signature("cannot open /var/x.txt: denied")
12563        );
12564        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12565    }
12566
12567    #[test]
12568    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12569        let other = FailClass::Other("x".into());
12570        assert!(should_hand_over(None, &FailClass::Quota));
12571        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12572        assert!(should_hand_over(
12573            Some(&FailClass::Timeout),
12574            &FailClass::Timeout
12575        ));
12576        assert!(should_hand_over(None, &other));
12577        assert!(!should_hand_over(Some(&other), &other));
12578        assert!(should_hand_over(
12579            Some(&other),
12580            &FailClass::Other("y".into())
12581        ));
12582        // A quota or timeout in between ends the run of identical failures.
12583        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12584        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12585    }
12586
12587    #[test]
12588    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12589        let a = SeatState::new("judge-1", "alpha", 7);
12590        let b = handover_seat("judge-1", "beta", 7);
12591        assert_ne!(a.claude_session, b.claude_session);
12592        assert_eq!(b.turns, 0);
12593    }
12594
12595    #[test]
12596    fn a_timeout_is_classified_apart_from_other_failures() {
12597        assert_eq!(
12598            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12599            Some(FailClass::Timeout)
12600        );
12601        assert!(matches!(
12602            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12603            Some(FailClass::Other(_))
12604        ));
12605    }
12606
12607    #[test]
12608    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12609        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12610        let tried = BTreeSet::from(["beta".to_owned()]);
12611        // beta sits at index 1; the next candidate is gamma, never alpha —
12612        // which is very likely a different candidate slot's own agent.
12613        let next = next_untried_in_roster(&roster, 1, &tried);
12614        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12615    }
12616
12617    #[test]
12618    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12619        let roster = vec![spec("alpha"), spec("beta")];
12620        let tried = BTreeSet::from(["beta".to_owned()]);
12621        // beta is the roster's last entry: nothing follows it, and alpha —
12622        // earlier in the roster, almost certainly a different candidate
12623        // slot's own agent — must not be reached by wrapping back to it.
12624        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12625    }
12626
12627    #[test]
12628    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12629        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12630        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12631        // beta (index 1) and gamma (index 2, the only entry after it) have
12632        // both been tried; alpha (index 0) never has, but it comes before
12633        // beta's own position, so there is nothing further for this seat.
12634        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12635    }
12636
12637    #[test]
12638    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12639        let roster = vec![spec("a"), spec("a"), spec("b")];
12640        let tried = BTreeSet::from(["a".to_owned()]);
12641        let next = next_untried_in_roster(&roster, 0, &tried);
12642        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12643    }
12644
12645    #[test]
12646    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12647        let roster = vec![spec("a"), spec("b")];
12648        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12649        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12650    }
12651
12652    #[test]
12653    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12654        let dir = tempfile::tempdir().unwrap();
12655        let bay = dir.path().join("ffff");
12656
12657        // Not there yet: nothing to do, nothing to panic on.
12658        remove_if_empty(&bay);
12659        assert!(!bay.exists());
12660
12661        // Something still inside - the winner's worktree, or a stray file -
12662        // keeps the directory standing.
12663        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12664        remove_if_empty(&bay);
12665        assert!(bay.exists(), "non-empty directory must survive");
12666
12667        // Once the last entry is gone, so is the directory itself.
12668        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12669        remove_if_empty(&bay);
12670        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12671    }
12672
12673    // `round_is_clean` is the exact decision this task fixed: a round with a
12674    // seat that never answered must not read the same as a round every seat
12675    // actually reviewed. These are deterministic and process-free by design —
12676    // the equivalent end-to-end check (a real reviewer timing out under a
12677    // live graph run) is a genuine race against wall-clock contention, and a
12678    // spawn slow enough to blow even a generous budget under a loaded test
12679    // run must not turn this specific regression check flaky.
12680
12681    #[test]
12682    fn a_full_panel_that_found_nothing_is_clean() {
12683        assert!(round_is_clean(
12684            0,
12685            true,
12686            2,
12687            2,
12688            0,
12689            IncompleteReviewPolicy::Block
12690        ));
12691    }
12692
12693    #[test]
12694    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12695        assert!(!round_is_clean(
12696            0,
12697            true,
12698            1,
12699            2,
12700            0,
12701            IncompleteReviewPolicy::Block
12702        ));
12703    }
12704
12705    #[test]
12706    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12707        assert!(!round_is_clean(
12708            1,
12709            true,
12710            1,
12711            2,
12712            0,
12713            IncompleteReviewPolicy::Warn
12714        ));
12715    }
12716
12717    #[test]
12718    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12719        assert!(round_is_clean(
12720            0,
12721            true,
12722            1,
12723            2,
12724            0,
12725            IncompleteReviewPolicy::Warn
12726        ));
12727    }
12728
12729    #[test]
12730    fn a_full_panel_with_an_open_finding_is_not_clean() {
12731        assert!(!round_is_clean(
12732            1,
12733            true,
12734            2,
12735            2,
12736            0,
12737            IncompleteReviewPolicy::Block
12738        ));
12739    }
12740
12741    #[test]
12742    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12743        assert!(!round_is_clean(
12744            0,
12745            false,
12746            2,
12747            2,
12748            0,
12749            IncompleteReviewPolicy::Block
12750        ));
12751    }
12752
12753    // The stall this task closes: under the default `block` policy, a seat
12754    // missing only because it was rate limited must not force a wait for a
12755    // session limit that will not lift by the next round. `round_is_clean`
12756    // is where that quorum carve-out lives; the review loop around it never
12757    // changes what a reviewer's vote or a finding's severity means.
12758
12759    #[test]
12760    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12761        // 1 of 2 answered, and the one missing was quota'd — the exact
12762        // "review-2 rate limited (quota)" shape from the field report.
12763        assert!(round_is_clean(
12764            0,
12765            true,
12766            1,
12767            2,
12768            1,
12769            IncompleteReviewPolicy::Block
12770        ));
12771    }
12772
12773    #[test]
12774    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12775        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12776        // not a quota loss (`quota_missing` stays 0) — worth another try.
12777        assert!(!round_is_clean(
12778            0,
12779            true,
12780            1,
12781            2,
12782            0,
12783            IncompleteReviewPolicy::Block
12784        ));
12785    }
12786
12787    #[test]
12788    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12789        assert!(!round_is_clean(
12790            1,
12791            true,
12792            1,
12793            2,
12794            1,
12795            IncompleteReviewPolicy::Block
12796        ));
12797        assert!(!round_is_clean(
12798            0,
12799            false,
12800            1,
12801            2,
12802            1,
12803            IncompleteReviewPolicy::Block
12804        ));
12805    }
12806
12807    #[test]
12808    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12809        // Every seat quota'd, nobody answered: there is no panel to decide
12810        // on, so this must fall through to the existing block-and-retry
12811        // fallback rather than call an unreviewed patch clean.
12812        assert!(!round_is_clean(
12813            0,
12814            true,
12815            0,
12816            2,
12817            2,
12818            IncompleteReviewPolicy::Block
12819        ));
12820    }
12821
12822    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12823        CommandOutcome {
12824            command: "test".to_owned(),
12825            code,
12826            output_tail: String::new(),
12827            duration_ms: 0,
12828            resource_blocked,
12829        }
12830    }
12831
12832    #[test]
12833    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12834        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12835        assert!(
12836            !verify_inconclusive(&[outcome(Some(1), false)]),
12837            "an ordinary failure is still evidence about the patch"
12838        );
12839        assert!(verify_inconclusive(&[outcome(None, true)]));
12840        assert!(
12841            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12842            "one inconclusive outcome taints the whole batch"
12843        );
12844        assert!(!verify_inconclusive(&[]));
12845    }
12846
12847    #[tokio::test]
12848    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12849        // Alive for the first two checks, then dead - confirms the loop
12850        // actually re-polls rather than deciding once and sleeping out the
12851        // ceiling regardless.
12852        let calls = std::sync::atomic::AtomicUsize::new(0);
12853        let started = Instant::now();
12854        wait_for_pids_with(
12855            &[123],
12856            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12857            Duration::from_millis(5),
12858            Duration::from_secs(5),
12859        )
12860        .await;
12861        assert!(
12862            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12863            "must keep checking rather than deciding on the first answer"
12864        );
12865        assert!(
12866            started.elapsed() < Duration::from_secs(1),
12867            "must return the moment it is confirmed dead, not wait out the ceiling"
12868        );
12869    }
12870
12871    #[tokio::test]
12872    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12873        let started = Instant::now();
12874        wait_for_pids_with(
12875            &[123],
12876            |_| true, // never reports dead
12877            Duration::from_millis(5),
12878            Duration::from_millis(30),
12879        )
12880        .await;
12881        let elapsed = started.elapsed();
12882        assert!(
12883            elapsed >= Duration::from_millis(30),
12884            "must not give up before its own ceiling: {elapsed:?}"
12885        );
12886        assert!(
12887            elapsed < Duration::from_secs(1),
12888            "must not wait past its own ceiling either: {elapsed:?}"
12889        );
12890    }
12891
12892    #[tokio::test]
12893    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12894        let started = Instant::now();
12895        wait_for_pids_with(
12896            &[],
12897            |_| true,
12898            Duration::from_secs(5),
12899            Duration::from_secs(5),
12900        )
12901        .await;
12902        assert!(
12903            started.elapsed() < Duration::from_millis(200),
12904            "an empty pid list has nothing to confirm"
12905        );
12906    }
12907
12908    // `review_conclusion` is the exact decision the review hand-off task
12909    // fixed: a round budget spent (or a tree that stopped moving) must not
12910    // collapse into `Blocked` regardless of what verification actually
12911    // said. Deterministic and process-free for the same reason the
12912    // `round_is_clean` family above is.
12913    fn review_round(
12914        clean: bool,
12915        blocking: usize,
12916        answered: usize,
12917        expected: usize,
12918        progressed: bool,
12919        e2e_ok: bool,
12920    ) -> ReviewRound {
12921        ReviewRound {
12922            round: 1,
12923            head: "h".to_owned(),
12924            verified_head: None,
12925            verified_at: None,
12926            reviews: Vec::new(),
12927            e2e: vec![CommandOutcome {
12928                command: "test".to_owned(),
12929                code: Some(if e2e_ok { 0 } else { 1 }),
12930                output_tail: String::new(),
12931                duration_ms: 0,
12932                resource_blocked: false,
12933            }],
12934            verify_retried: false,
12935            e2e_deferred: false,
12936            e2e_defer_reason: None,
12937            fix: None,
12938            blocking,
12939            answered,
12940            expected,
12941            clean,
12942            progressed,
12943            vote_split: false,
12944            reconsideration: Vec::new(),
12945            verdict: None,
12946        }
12947    }
12948
12949    #[test]
12950    fn review_conclusion_is_none_when_nothing_has_run() {
12951        assert_eq!(review_conclusion(&[], 3), None);
12952    }
12953
12954    #[test]
12955    fn review_conclusion_is_none_while_rounds_remain() {
12956        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12957        assert_eq!(review_conclusion(&rounds, 3), None);
12958    }
12959
12960    #[test]
12961    fn review_conclusion_is_gating_once_a_round_is_clean() {
12962        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12963        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12964    }
12965
12966    #[test]
12967    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12968        let rounds = vec![
12969            review_round(false, 1, 2, 2, true, true),
12970            review_round(false, 1, 2, 2, true, true),
12971        ];
12972        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12973    }
12974
12975    #[test]
12976    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12977        let rounds = vec![
12978            review_round(false, 1, 2, 2, true, true),
12979            review_round(false, 1, 2, 2, true, false),
12980        ];
12981        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12982    }
12983
12984    #[test]
12985    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12986        // Magi never got a command to run against this round's own head — a
12987        // resource-blocked attempt, not a red one — so this must never
12988        // settle on `Blocked` the way a genuine e2e failure would. `None`
12989        // here is what tells `Runner::review_loop` to retry the check
12990        // itself rather than trust this cheap recomputation with a verdict
12991        // it cannot actually produce.
12992        let mut blocked = review_round(false, 1, 2, 2, true, false);
12993        blocked.e2e[0].resource_blocked = true;
12994        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12995        assert_eq!(review_conclusion(&rounds, 2), None);
12996    }
12997
12998    #[test]
12999    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13000        // Missing input, not a verified tree — never a hand-off candidate.
13001        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13002        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13003    }
13004
13005    #[test]
13006    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13007        let rounds = vec![
13008            review_round(false, 1, 2, 2, false, true),
13009            review_round(false, 1, 2, 2, false, true),
13010        ];
13011        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13012    }
13013
13014    fn secs(n: u64) -> Duration {
13015        Duration::from_secs(n)
13016    }
13017
13018    /// A throwaway repo with one commit on `main`, for tests that need `merge`
13019    /// to make real (and, if it runs at all, real*ly fail*) git calls.
13020    #[tokio::test]
13021    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13022        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13023        if std::env::var_os(CHILD).is_none() {
13024            // Other tests reconcile every run in the process-global home,
13025            // briefly taking owner locks. This ancestry test expects an idle
13026            // predecessor, so give it its own process and pinned test home.
13027            let output = std::process::Command::new(std::env::current_exe().unwrap())
13028                .args([
13029                    "--exact",
13030                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13031                    "--nocapture",
13032                ])
13033                .env(CHILD, "1")
13034                .env_remove("MAGI_WORKER_HOME")
13035                .env_remove("MAGI_WORKER_ID")
13036                .output()
13037                .unwrap();
13038            assert!(
13039                output.status.success()
13040                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13041                "{}\n{}",
13042                String::from_utf8_lossy(&output.stdout),
13043                String::from_utf8_lossy(&output.stderr)
13044            );
13045            return;
13046        }
13047        ask_test_home();
13048        let tmp = tempfile::tempdir().unwrap();
13049        let repo = tmp.path().join("repo");
13050        std::fs::create_dir_all(&repo).unwrap();
13051        init_repo(&repo);
13052        let g = |args: &[&str]| {
13053            let out = std::process::Command::new("git")
13054                .args(args)
13055                .current_dir(&repo)
13056                .quiet()
13057                .output()
13058                .unwrap();
13059            assert!(
13060                out.status.success(),
13061                "{}",
13062                String::from_utf8_lossy(&out.stderr)
13063            );
13064        };
13065        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13066        g(&["checkout", "-b", "historical"]);
13067        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13068        g(&["add", "."]);
13069        g(&["commit", "-m", "old alternative"]);
13070        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13071        g(&["checkout", "-b", "magi/task-selected", &base]);
13072        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13073        g(&["add", "."]);
13074        g(&["commit", "-m", "measurement infrastructure"]);
13075        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13076        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13077        g(&["add", "."]);
13078        g(&["commit", "-m", "dependent implementation"]);
13079        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13080        g(&["checkout", "main"]);
13081        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13082        g(&["add", "."]);
13083        g(&["commit", "-m", "unrelated base change"]);
13084        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13085        let origin = tmp.path().join("origin.git");
13086        g(&[
13087            "clone",
13088            "--bare",
13089            repo.to_str().unwrap(),
13090            origin.to_str().unwrap(),
13091        ]);
13092        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13093        g(&["fetch", "origin"]);
13094        let selection = crate::pr_recovery::Selection {
13095            url: "https://example.test/project/pull/1".into(),
13096            branch: "magi/task-selected".into(),
13097            head: head.clone(),
13098            source_head: None,
13099            source_run: None,
13100            source_proof: None,
13101            disposition: Some(crate::pr_recovery::Disposition::Review),
13102            instruction: "preserve measured behavior".into(),
13103            approved_at: jiff::Timestamp::now(),
13104        };
13105        let mut template = runner_at(RunStatus::Reviewing);
13106        template.state.repo = repo.clone();
13107        template.state.base_commit = current.clone();
13108        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13109        template.state.origin = Some(crate::run::Origin::queue("selected"));
13110        template.state.pr_head = Some(selection.branch.clone());
13111        template.state.pr_recovery = Some(selection.clone());
13112        let mut runner =
13113            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13114                .await
13115                .unwrap();
13116        let wt = runner.state.winner().unwrap().worktree.clone();
13117        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13118        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13119        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13120        assert!(runner.state.seeds.is_empty());
13121        assert_eq!(runner.state.instruction, "preserve measured behavior");
13122        runner.sync_to_base().await.unwrap();
13123        assert_eq!(
13124            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13125            "prerequisite"
13126        );
13127        assert_eq!(
13128            std::fs::read_to_string(wt.join("implementation")).unwrap(),
13129            "measured implementation"
13130        );
13131        assert_eq!(
13132            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13133            "current main change"
13134        );
13135        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13136        assert_eq!(
13137            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13138            head
13139        );
13140
13141        // A failed selected-source setup stays inspectable, never resumable as a new competition.
13142        let mut partial = runner_at(RunStatus::Reviewing);
13143        partial.state.repo = repo.clone();
13144        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13145        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13146            run: Some("exit 1".into()),
13147            ..Default::default()
13148        }];
13149        let selection = runner.state.pr_recovery.clone().unwrap();
13150        partial.state.pr_recovery = Some(selection.clone());
13151        partial.state.pr_head = Some(selection.branch.clone());
13152        partial.state.origin = Some(crate::run::Origin::queue("selected"));
13153        let id = partial.state.id.clone();
13154        let error = match Runner::open_selected(
13155            &repo,
13156            partial.state,
13157            partial.roles,
13158            1,
13159            runner.state.base_commit.clone(),
13160            selection,
13161        )
13162        .await
13163        {
13164            Ok(_) => panic!("setup must fail"),
13165            Err(e) => e,
13166        };
13167        assert_eq!(
13168            error
13169                .downcast_ref::<crate::pr_recovery::StartupFailed>()
13170                .unwrap()
13171                .run,
13172            id
13173        );
13174        let failed = RunState::load(&id).unwrap();
13175        assert_eq!(failed.status, RunStatus::Blocked);
13176        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13177        assert!(Runner::resume(&id).is_err());
13178        runner.state.status = RunStatus::Ready;
13179        runner.state.driver_exited = true;
13180        runner.state.save().unwrap();
13181        let mut retry = runner_at(RunStatus::Reviewing);
13182        retry.state.repo = repo.clone();
13183        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13184        let mut reselected = runner.state.pr_recovery.clone().unwrap();
13185        reselected.approved_at = jiff::Timestamp::now();
13186        retry.state.origin = Some(crate::run::Origin::queue("selected"));
13187        retry.state.pr_head = Some(reselected.branch.clone());
13188        retry.state.pr_recovery = Some(reselected.clone());
13189        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13190            .await
13191            .unwrap();
13192        let restarted = Runner::open_selected(
13193            &repo,
13194            retry.state,
13195            retry.roles,
13196            1,
13197            runner.state.base_commit.clone(),
13198            reselected,
13199        )
13200        .await
13201        .unwrap();
13202        assert_eq!(
13203            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13204                .await
13205                .unwrap(),
13206            head
13207        );
13208        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13209        assert_eq!(
13210            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13211            head
13212        );
13213    }
13214
13215    #[tokio::test]
13216    #[cfg(unix)]
13217    async fn fallback_and_interrupted_restoration_require_coordination() {
13218        use std::os::fd::AsRawFd as _;
13219        ask_test_home();
13220        let tmp = tempfile::tempdir().unwrap();
13221        let mut runner = registered_runner(tmp.path()).await;
13222        runner.state.config.graph.timeout_resource_wait = 1;
13223        let winner = runner.state.candidates[0].clone();
13224        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13225        let leftover = winner.worktree.join("leftover");
13226        std::fs::write(&leftover, "untrusted edits").unwrap();
13227        let next = tmp.path().join("next-fixer");
13228        let mut failed = spec("first");
13229        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13230        let mut successor = spec("second");
13231        successor.command = vec![
13232            "sh".into(),
13233            "-c".into(),
13234            format!("touch '{}'; echo done", next.display()),
13235        ];
13236        let attempts = vec![
13237            (failed.clone(), "first".into()),
13238            (successor.clone(), "second".into()),
13239        ];
13240        let lock_path = runner.state.repo.join(".git/updater.lock");
13241        let external = std::fs::File::create(&lock_path).unwrap();
13242        assert_eq!(
13243            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13244            0
13245        );
13246        let artifacts = tmp.path().join("artifacts");
13247        let build = |spec, seat| SeatJob {
13248            spec,
13249            seat,
13250            cwd: winner.worktree.clone(),
13251            prompt: "fixture".into(),
13252            timeout: Duration::from_secs(10),
13253            allow_write: true,
13254            sessions: false,
13255            artifacts: artifacts.clone(),
13256            stem: "fix".into(),
13257            handover: None,
13258        };
13259        let (_, _, out) = runner
13260            .ask_fixer_among(
13261                attempts.clone(),
13262                "graft",
13263                None,
13264                Some((&winner.worktree, &head)),
13265                build,
13266            )
13267            .await;
13268        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13269        assert!(leftover.exists(), "contention must prevent clean");
13270        assert!(!next.exists());
13271        assert!(runner.state.handovers.is_empty());
13272        // Early interrupted-repair rollback uses the same protected route.
13273        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13274            from_run: "earlier".into(),
13275            generation: 1,
13276            mode: crate::recovery::Mode::VerifyRepair,
13277            reasons: Vec::new(),
13278            reviewed_head: None,
13279            rounds: Vec::new(),
13280            open_findings: Vec::new(),
13281            evidence: None,
13282            criteria_inherited: true,
13283            repair: crate::recovery::Repair::Started {
13284                before: head.clone(),
13285            },
13286        });
13287        assert!(
13288            runner
13289                .recovery_repair(&winner)
13290                .await
13291                .unwrap_err()
13292                .is::<crate::repo_lock::ResourceBlocked>()
13293        );
13294        assert!(matches!(
13295            runner.state.recovery.as_ref().unwrap().repair,
13296            crate::recovery::Repair::Started { .. }
13297        ));
13298        assert!(leftover.exists());
13299        drop(external);
13300        runner.recovery_repair(&winner).await.unwrap();
13301        assert!(!leftover.exists());
13302        assert!(matches!(
13303            runner.state.recovery.as_ref().unwrap().repair,
13304            crate::recovery::Repair::Done { .. }
13305        ));
13306        // A hard lock-open failure also leaves HEAD and untracked work intact.
13307        std::fs::remove_file(&lock_path).unwrap();
13308        std::fs::create_dir(&lock_path).unwrap();
13309        std::fs::write(&leftover, "untrusted edits").unwrap();
13310        let (_, _, out) = runner
13311            .ask_fixer_among(
13312                attempts,
13313                "graft",
13314                None,
13315                Some((&winner.worktree, &head)),
13316                build,
13317            )
13318            .await;
13319        assert!(matches!(out, AgentOutcome::Failed(_)));
13320        assert!(leftover.exists());
13321        assert!(!next.exists());
13322        assert_eq!(
13323            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13324            head
13325        );
13326    }
13327
13328    #[cfg(unix)]
13329    async fn registered_runner(root: &Path) -> Runner {
13330        ask_test_home();
13331        let repo = root.join("repo");
13332        std::fs::create_dir_all(&repo).unwrap();
13333        init_repo(&repo);
13334        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13335        let mut runner = runner_at(RunStatus::Prep);
13336        runner.state.repo = repo.clone();
13337        runner.state.config.coordination = crate::config::test_coordination();
13338        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13339        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13340        runner.state.config.graph.advise = false;
13341        runner.roles.implementers = vec![spec("implement")];
13342        runner.roles.judges = vec![spec("judge")];
13343        runner.prep().await.unwrap();
13344        runner
13345    }
13346
13347    #[tokio::test]
13348    #[cfg(unix)]
13349    async fn parked_implementer_persists_completed_work() {
13350        for registered in [false, true] {
13351            let tmp = tempfile::tempdir().unwrap();
13352            let mut runner = registered_runner(tmp.path()).await;
13353            if !registered {
13354                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13355            }
13356            let marker = tmp.path().join("started");
13357            let release = tmp.path().join("release");
13358            let mut implementer = spec("implement");
13359            implementer.command = vec![
13360                "sh".into(),
13361                "-c".into(),
13362                format!(
13363                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13364                    marker.display(),
13365                    release.display()
13366                ),
13367            ];
13368            runner.roles.implementers = vec![implementer.clone()];
13369            runner.state.config.agents = vec![implementer];
13370            let pause = runner.pause.clone();
13371            let (result, ()) = tokio::join!(runner.implement(), async {
13372                tokio::time::timeout(Duration::from_secs(10), async {
13373                    while !marker.exists() {
13374                        tokio::time::sleep(Duration::from_millis(10)).await;
13375                    }
13376                })
13377                .await
13378                .unwrap();
13379                pause.park();
13380                std::fs::write(&release, "go").unwrap();
13381            });
13382            result.unwrap();
13383            let candidate = &runner.state.candidates[0];
13384            assert!(candidate.commits > 0, "{candidate:?}");
13385            assert!(candidate.summary.contains("saved-result"));
13386            assert_eq!(candidate.files, 1);
13387            let before = candidate.commits;
13388            runner.implement().await.unwrap();
13389            assert_eq!(runner.state.candidates[0].commits, before);
13390        }
13391    }
13392
13393    #[tokio::test]
13394    #[cfg(unix)]
13395    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13396        let tmp = tempfile::tempdir().unwrap();
13397        let mut runner = registered_runner(tmp.path()).await;
13398        let winner = runner.state.candidates[0].clone();
13399        let cache = tmp.path().join("cache");
13400        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13401        runner.state.config.graph.timeout_resource_wait = 0;
13402        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13403        let crate::cache::AcquireOutcome::Acquired(held) =
13404            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13405        else {
13406            panic!("cache must be free")
13407        };
13408        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13409            from_run: "earlier".into(),
13410            generation: 1,
13411            mode: crate::recovery::Mode::VerifyRepair,
13412            reasons: Vec::new(),
13413            reviewed_head: None,
13414            rounds: Vec::new(),
13415            open_findings: Vec::new(),
13416            evidence: None,
13417            criteria_inherited: true,
13418            repair: crate::recovery::Repair::NotStarted,
13419        });
13420        runner.recovery_repair(&winner).await.unwrap();
13421        assert!(matches!(
13422            runner.state.recovery.as_ref().unwrap().repair,
13423            crate::recovery::Repair::WaitingForRestore { .. }
13424        ));
13425        drop(held);
13426        runner.state.resource_block = None;
13427        runner.state.config.graph.timeout_resource_wait = 1;
13428        runner.recovery_repair(&winner).await.unwrap();
13429        assert!(matches!(
13430            runner.state.recovery.as_ref().unwrap().repair,
13431            crate::recovery::Repair::Done { .. }
13432        ));
13433        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13434    }
13435
13436    #[tokio::test]
13437    #[cfg(unix)]
13438    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13439        use std::os::fd::AsRawFd as _;
13440        for graft in [false, true] {
13441            let tmp = tempfile::tempdir().unwrap();
13442            let mut runner = registered_runner(tmp.path()).await;
13443            runner.state.config.graph.timeout_resource_wait = 1;
13444            let winner = runner.state.candidates[0].clone();
13445            let marker = tmp.path().join("fixer-started");
13446            let mut fixer = spec("fixer");
13447            fixer.command = vec![
13448                "sh".into(),
13449                "-c".into(),
13450                format!("echo started >> '{}'; echo done", marker.display()),
13451            ];
13452            runner.roles.fixer = Some(vec![fixer]);
13453            if graft {
13454                let mut loser = winner.clone();
13455                loser.label = 'B';
13456                loser.index = 1;
13457                runner.state.candidates.push(loser);
13458                runner.state.config.graph.graft_runner_up = true;
13459                runner.state.tally = Some(Tally {
13460                    first_choice: BTreeMap::new(),
13461                    borda: BTreeMap::new(),
13462                    winner: winner.label,
13463                    rankings: 1,
13464                    unanimous_initial: true,
13465                    deliberated: false,
13466                    changed_votes: 0,
13467                    unanimous_final: true,
13468                    tie_break: None,
13469                    judges: 1,
13470                    present: 1,
13471                    quorum: 1,
13472                    met_quorum: true,
13473                    uncontested: None,
13474                });
13475                runner.state.judgements.push(crate::run::Judgement {
13476                    judge: 1,
13477                    seat: "judge-1".into(),
13478                    agent: "judge".into(),
13479                    ranking: vec![winner.label, 'B'],
13480                    reasons: BTreeMap::new(),
13481                    confidence: None,
13482                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13483                    order: vec![0, 1],
13484                    failed: None,
13485                    duration_ms: 0,
13486                });
13487            } else {
13488                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13489                    from_run: "earlier".into(),
13490                    generation: 1,
13491                    mode: crate::recovery::Mode::VerifyRepair,
13492                    reasons: Vec::new(),
13493                    reviewed_head: None,
13494                    rounds: Vec::new(),
13495                    open_findings: Vec::new(),
13496                    evidence: None,
13497                    criteria_inherited: true,
13498                    repair: crate::recovery::Repair::NotStarted,
13499                });
13500            }
13501            let reader = crate::repo_lock::SeatLease::for_checkout(
13502                &winner.worktree,
13503                false,
13504                crate::config::test_coordination().as_ref(),
13505            )
13506            .await
13507            .unwrap();
13508            let pause = runner.pause.clone();
13509            let (result, ()) = tokio::join!(
13510                async {
13511                    if graft {
13512                        runner.graft().await
13513                    } else {
13514                        runner.recovery_repair(&winner).await
13515                    }
13516                },
13517                async {
13518                    tokio::time::sleep(Duration::from_millis(100)).await;
13519                    pause.park();
13520                }
13521            );
13522            result.unwrap();
13523            assert!(!marker.exists(), "a blocked seat must not spawn");
13524            // Exercise the persisted state, as a binary replacement/resume does.
13525            runner.state = RunState::load(&runner.state.id).unwrap();
13526            if graft {
13527                assert_eq!(
13528                    runner.state.graft.as_ref().unwrap().outcome,
13529                    GraftOutcome::WaitingForRestore
13530                );
13531            } else {
13532                assert!(matches!(
13533                    runner.state.recovery.as_ref().unwrap().repair,
13534                    crate::recovery::Repair::WaitingForRestore { .. }
13535                ));
13536            }
13537            drop(reader);
13538            runner.pause = Pause::new();
13539            runner.state.resource_block = None;
13540            // Deferred restoration must still respect an external lock owner.
13541            let external =
13542                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13543            assert_eq!(
13544                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13545                0
13546            );
13547            let leftover = winner.worktree.join("leftover");
13548            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13549            let result = if graft {
13550                runner.graft().await
13551            } else {
13552                runner.recovery_repair(&winner).await
13553            };
13554            assert!(
13555                result
13556                    .unwrap_err()
13557                    .is::<crate::repo_lock::ResourceBlocked>()
13558            );
13559            assert!(leftover.exists());
13560            assert!(!marker.exists());
13561            drop(external);
13562            runner.state.resource_block = None;
13563            let result = if graft {
13564                runner.graft().await
13565            } else {
13566                runner.recovery_repair(&winner).await
13567            };
13568            result.unwrap();
13569            assert!(!leftover.exists(), "restore must precede retry");
13570            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13571            if graft {
13572                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13573            } else {
13574                assert!(matches!(
13575                    runner.state.recovery.as_ref().unwrap().repair,
13576                    crate::recovery::Repair::Done { .. }
13577                ));
13578            }
13579        }
13580    }
13581
13582    #[tokio::test]
13583    #[cfg(unix)]
13584    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13585        use std::os::fd::AsRawFd as _;
13586        let tmp = tempfile::tempdir().unwrap();
13587        let mut runner = registered_runner(tmp.path()).await;
13588        runner.state.config.graph.timeout_resource_wait = 1;
13589        let winner = runner.state.candidates[0].clone();
13590        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13591        runner.state.tally = Some(Tally {
13592            first_choice: BTreeMap::new(),
13593            borda: BTreeMap::new(),
13594            winner: winner.label,
13595            rankings: 0,
13596            unanimous_initial: false,
13597            deliberated: false,
13598            changed_votes: 0,
13599            unanimous_final: false,
13600            tie_break: None,
13601            judges: 0,
13602            present: 0,
13603            quorum: 0,
13604            met_quorum: true,
13605            uncontested: Some("fixture".into()),
13606        });
13607        runner.state.graft = Some(GraftRecord {
13608            pre: pre.clone(),
13609            post: None,
13610            lines: None,
13611            limit: None,
13612            agent: None,
13613            outcome: GraftOutcome::Running,
13614            requested: Vec::new(),
13615            adoption: None,
13616            adoption_error: None,
13617        });
13618        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13619        git::commit_all(&winner.worktree, "unfinished graft")
13620            .await
13621            .unwrap();
13622        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13623        let leftover = winner.worktree.join("leftover");
13624        std::fs::write(&leftover, "untrusted edits").unwrap();
13625        let path = runner.state.repo.join(".git/updater.lock");
13626        let external = std::fs::File::create(&path).unwrap();
13627        assert_eq!(
13628            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13629            0
13630        );
13631        assert!(
13632            runner
13633                .graft()
13634                .await
13635                .unwrap_err()
13636                .is::<crate::repo_lock::ResourceBlocked>()
13637        );
13638        assert!(matches!(
13639            runner.state.graft.as_ref().unwrap().outcome,
13640            GraftOutcome::Running
13641        ));
13642        assert_eq!(
13643            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13644            unfinished
13645        );
13646        assert!(leftover.exists());
13647        drop(external);
13648        std::fs::remove_file(&path).unwrap();
13649        std::fs::create_dir(&path).unwrap();
13650        assert!(runner.graft().await.is_err());
13651        assert!(matches!(
13652            runner.state.graft.as_ref().unwrap().outcome,
13653            GraftOutcome::Running
13654        ));
13655        assert_eq!(
13656            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13657            unfinished
13658        );
13659        assert!(leftover.exists());
13660        std::fs::remove_dir(&path).unwrap();
13661        runner.graft().await.unwrap();
13662        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13663        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13664        assert!(!leftover.exists());
13665    }
13666
13667    #[tokio::test]
13668    #[cfg(unix)]
13669    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13670        use std::os::fd::AsRawFd as _;
13671        let tmp = tempfile::tempdir().unwrap();
13672        let mut runner = registered_runner(tmp.path()).await;
13673        runner.state.prep_complete = Some(false);
13674        runner.state.config.graph.timeout_resource_wait = 1;
13675        runner.state.save().unwrap();
13676        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13677        assert_eq!(
13678            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13679            0
13680        );
13681        runner.execute().await.unwrap();
13682        let disk = RunState::load(&runner.state.id).unwrap();
13683        assert!(disk.driver_exited);
13684        assert_eq!(disk.driver_pid, None);
13685        assert_eq!(runner.state.status, RunStatus::Blocked);
13686        assert_eq!(
13687            runner.state.resource_block.as_ref().unwrap().kind,
13688            "repository coordination"
13689        );
13690        assert!(runner.state.prep_failure.is_none());
13691        assert!(runner.state.seats.is_empty());
13692        runner.pause.park();
13693        // Include headroom for Git subprocesses under parallel test load;
13694        // the repository contention deadline itself remains one second.
13695        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13696            .await
13697            .unwrap()
13698            .unwrap();
13699        assert!(runner.state.parked);
13700        assert!(runner.state.prep_failure.is_none());
13701        let disk = RunState::load(&runner.state.id).unwrap();
13702        assert!(disk.driver_exited);
13703        assert_eq!(disk.driver_pid, None);
13704    }
13705
13706    #[tokio::test]
13707    #[cfg(unix)]
13708    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13709        use std::os::fd::AsRawFd as _;
13710        let tmp = tempfile::tempdir().unwrap();
13711        let mut runner = registered_runner(tmp.path()).await;
13712        runner.state.prep_complete = Some(false);
13713        runner.state.config.graph.timeout_resource_wait = 30;
13714        runner.state.save().unwrap();
13715        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13716        assert_eq!(
13717            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13718            0
13719        );
13720        assert!(
13721            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13722                .await
13723                .is_err()
13724        );
13725        let disk = RunState::load(&runner.state.id).unwrap();
13726        assert!(disk.driver_exited);
13727        assert_eq!(disk.driver_pid, None);
13728        assert!(disk.status.resumable());
13729        assert!(runner.state.driver_exited);
13730        assert_eq!(runner.state.driver_pid, None);
13731        runner.state.save().unwrap();
13732        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13733    }
13734
13735    #[tokio::test]
13736    #[cfg(unix)]
13737    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13738        let tmp = tempfile::tempdir().unwrap();
13739        let mut runner = registered_runner(tmp.path()).await;
13740        let winner = runner.state.candidates[0].clone();
13741        let cache = tmp.path().join("cache");
13742        let started = tmp.path().join("pre-gate-started");
13743        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13744        runner.state.config.verify.gate =
13745            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13746        let home = crate::run::home();
13747        let owner =
13748            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13749        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13750            crate::cache::AcquireOutcome::Acquired(g) => g,
13751            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13752        };
13753        let mut gate = Box::pin(runner.run_gate(&winner));
13754        assert!(
13755            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13756                .await
13757                .is_err()
13758        );
13759        assert!(!started.exists(), "pre-gate must wait for the cache too");
13760        // Simulate the wave that already owns the cache entering invoke.
13761        // It must obtain the editor reservation and finish without waiting
13762        // for the gate that is queued behind its cache.
13763        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13764        let editor = crate::repo_lock::SeatLease::checkout_with(
13765            &winner.worktree,
13766            true,
13767            crate::config::test_coordination().as_ref(),
13768            &wait,
13769        )
13770        .await
13771        .unwrap();
13772        drop(editor);
13773        guard.release();
13774        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13775            .await
13776            .unwrap()
13777            .unwrap();
13778        drop(gate);
13779        assert!(outcomes.iter().all(CommandOutcome::ok));
13780        assert!(started.exists());
13781    }
13782
13783    #[tokio::test]
13784    #[cfg(unix)]
13785    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13786        use std::os::fd::AsRawFd as _;
13787
13788        ask_test_home();
13789        let tmp = tempfile::tempdir().unwrap();
13790        let repo = tmp.path().join("repo");
13791        std::fs::create_dir_all(&repo).unwrap();
13792        init_repo(&repo);
13793        let common = repo.join(".git");
13794        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13795        let mut runner = runner_at(RunStatus::Prep);
13796        runner.state.repo = repo.clone();
13797        runner.state.config.coordination = crate::config::test_coordination();
13798        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13799        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13800        runner.state.config.graph.advise = false;
13801        runner.roles.implementers = vec![spec("implement")];
13802        runner.roles.judges = vec![spec("judge")];
13803        runner.prep().await.unwrap();
13804        let winner = runner.state.candidates[0].clone();
13805        // Emulate edits already left by a completed seat. The fixture CLI
13806        // only reports completion; it performs no repository operations.
13807        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13808        let completed = tmp.path().join("seat-completed");
13809        runner.roles.implementers[0].command = vec![
13810            "sh".into(),
13811            "-c".into(),
13812            format!("touch '{}'; echo completed", completed.display()),
13813        ];
13814        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13815        assert_eq!(
13816            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13817            0
13818        );
13819        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13820        let mut implement = Box::pin(runner.implement());
13821        tokio::select! {
13822            result = &mut implement => panic!("rescue must wait: {result:?}"),
13823            result = tokio::time::timeout(Duration::from_secs(10), async {
13824                while !completed.exists() {
13825                    tokio::time::sleep(Duration::from_millis(20)).await;
13826                }
13827            }) => result.unwrap(),
13828        }
13829        assert!(
13830            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13831                .await
13832                .is_err()
13833        );
13834        assert_eq!(
13835            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13836            before
13837        );
13838        assert_eq!(
13839            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13840            0
13841        );
13842        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13843            .await
13844            .unwrap()
13845            .unwrap();
13846        drop(implement);
13847        assert_ne!(
13848            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13849            before
13850        );
13851        assert!(git::is_clean(&winner.worktree).await.unwrap());
13852
13853        let started = tmp.path().join("pre-gate-started");
13854        let proceed = tmp.path().join("pre-gate-proceed");
13855        let checked = tmp.path().join("gate-ran");
13856        runner.state.config.verify.pre_gate = vec![format!(
13857            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13858            started.display(),
13859            proceed.display()
13860        )];
13861        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13862        assert_eq!(
13863            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13864            0
13865        );
13866        let mut gate = Box::pin(runner.run_gate(&winner));
13867        assert!(
13868            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13869                .await
13870                .is_err()
13871        );
13872        assert!(!started.exists());
13873        assert!(!checked.exists());
13874        assert!(git::is_clean(&winner.worktree).await.unwrap());
13875        assert_eq!(
13876            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13877            0
13878        );
13879        tokio::select! {
13880            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13881            result = tokio::time::timeout(Duration::from_secs(10), async {
13882                while !started.exists() {
13883                    tokio::time::sleep(Duration::from_millis(20)).await;
13884                }
13885            }) => result.unwrap(),
13886        }
13887        assert_eq!(
13888            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13889            -1,
13890            "mechanical edits must hold the real lock"
13891        );
13892        std::fs::write(proceed, "ready").unwrap();
13893        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13894            .await
13895            .unwrap()
13896            .unwrap();
13897        drop(gate);
13898        assert!(outcomes.iter().all(CommandOutcome::ok));
13899        assert!(checked.exists());
13900        assert!(runner.state.pre_gate_commit.is_some());
13901        assert!(git::is_clean(&winner.worktree).await.unwrap());
13902        assert_eq!(
13903            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13904            0,
13905            "gate completion must release coordination"
13906        );
13907        drop(outsider);
13908        std::fs::remove_file(common.join("updater.lock")).unwrap();
13909        std::fs::create_dir(common.join("updater.lock")).unwrap();
13910        std::fs::remove_file(&started).unwrap();
13911        std::fs::remove_file(&checked).unwrap();
13912        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13913        let error = runner.run_gate(&winner).await.unwrap_err();
13914        assert!(
13915            error
13916                .to_string()
13917                .contains("acquire mandatory repository lock")
13918        );
13919        assert!(
13920            !started.exists(),
13921            "failed acquisition must prevent pre-gate"
13922        );
13923        assert!(!checked.exists(), "failed acquisition must prevent gate");
13924        assert!(git::is_clean(&winner.worktree).await.unwrap());
13925        assert_eq!(
13926            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13927            head
13928        );
13929    }
13930
13931    #[tokio::test]
13932    #[cfg(unix)]
13933    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13934        use std::os::fd::AsRawFd as _;
13935        use std::time::Duration;
13936
13937        ask_test_home();
13938        let tmp = tempfile::tempdir().unwrap();
13939        let repo = tmp.path().join("repo");
13940        std::fs::create_dir_all(&repo).unwrap();
13941        init_repo(&repo);
13942        let common = repo.join(".git");
13943        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13944        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13945        assert_eq!(
13946            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13947            0
13948        );
13949        let mut runner = runner_at(RunStatus::Prep);
13950        runner.state.repo = repo.clone();
13951        runner.state.config.coordination = crate::config::test_coordination();
13952        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13953        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13954        runner.state.config.graph.advise = true;
13955        runner.state.config.graph.advisors = 1;
13956        runner.roles.implementers = vec![spec("implement")];
13957        runner.roles.judges = vec![spec("judge")];
13958        let root = runner.state.worktree_root();
13959        let started = tmp.path().join("setup-started");
13960        let proceed = tmp.path().join("setup-proceed");
13961        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13962            run: Some(format!(
13963                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13964                started.display(),
13965                proceed.display()
13966            )),
13967            ..Default::default()
13968        }];
13969        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13970        let config = git::git(&repo, &["config", "--local", "--list"])
13971            .await
13972            .unwrap();
13973        let mut prep = Box::pin(runner.prep());
13974        assert!(
13975            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13976                .await
13977                .is_err()
13978        );
13979        assert!(
13980            !root.exists(),
13981            "contention must prevent even checkout creation"
13982        );
13983        assert!(!started.exists(), "contention must prevent setup");
13984        assert_eq!(
13985            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13986            branches
13987        );
13988        assert_eq!(
13989            git::git(&repo, &["config", "--local", "--list"])
13990                .await
13991                .unwrap(),
13992            config
13993        );
13994        drop(lock);
13995        tokio::select! {
13996            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13997            result = tokio::time::timeout(Duration::from_secs(10), async {
13998                while !started.exists() {
13999                    tokio::time::sleep(Duration::from_millis(20)).await;
14000                }
14001            }) => result.unwrap(),
14002        }
14003        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14004        assert_eq!(
14005            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14006            -1
14007        );
14008        assert!(
14009            tokio::time::timeout(
14010                Duration::from_millis(100),
14011                crate::repo_lock::SeatLease::for_checkout(
14012                    &repo,
14013                    true,
14014                    crate::config::test_coordination().as_ref()
14015                )
14016            )
14017            .await
14018            .is_err(),
14019            "preparation must also exclude local editors"
14020        );
14021        std::fs::write(proceed, "ready").unwrap();
14022        tokio::time::timeout(Duration::from_secs(10), &mut prep)
14023            .await
14024            .unwrap()
14025            .unwrap();
14026        drop(prep);
14027        assert_eq!(runner.state.prep_complete, Some(true));
14028        assert!(root.join("judge-1").exists());
14029        assert!(root.join("advisor-1").exists());
14030        assert_eq!(
14031            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14032            0
14033        );
14034    }
14035
14036    #[tokio::test]
14037    #[cfg(unix)]
14038    async fn preparation_lock_failure_prevents_repository_changes() {
14039        ask_test_home();
14040        let tmp = tempfile::tempdir().unwrap();
14041        let repo = tmp.path().join("repo");
14042        std::fs::create_dir_all(&repo).unwrap();
14043        init_repo(&repo);
14044        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14045        // A directory at the required lock path makes opening the real lock
14046        // fail, rather than merely exercising an invalid registration.
14047        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14048        let mut runner = runner_at(RunStatus::Prep);
14049        runner.state.repo = repo.clone();
14050        runner.state.config.coordination = crate::config::test_coordination();
14051        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14052        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14053        runner.roles.implementers = vec![spec("implement")];
14054        let error = runner.prep().await.unwrap_err();
14055        assert!(
14056            error
14057                .to_string()
14058                .contains("acquire mandatory repository lock")
14059        );
14060        assert!(!runner.state.worktree_root().exists());
14061        assert!(runner.state.candidates.is_empty());
14062        assert!(!repo.join(".git/config.worktree").exists());
14063        assert_eq!(
14064            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14065            "* main"
14066        );
14067    }
14068
14069    #[tokio::test]
14070    async fn preparation_records_creation_intent_before_git_can_fail() {
14071        ask_test_home();
14072        let tmp = tempfile::tempdir().unwrap();
14073        let repo = tmp.path().join("repo");
14074        std::fs::create_dir_all(&repo).unwrap();
14075        init_repo(&repo);
14076        let mut runner = runner_at(RunStatus::Prep);
14077        runner.state.repo = repo.clone();
14078        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14079        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14080        runner.state.config.blind.commit_msg_hook = false;
14081        runner.roles.implementers = vec![spec("one")];
14082        let label = blind::assign_labels(1, runner.state.seed)[0];
14083        let branch = runner.state.branch_for(label);
14084        // A conflicting branch makes Git refuse creation before a candidate
14085        // can be registered. The durable intent must still identify it.
14086        git::git(&repo, &["branch", &branch]).await.unwrap();
14087        assert!(runner.prep().await.is_err());
14088        let saved = RunState::load(&runner.state.id).unwrap();
14089        assert!(saved.candidates.is_empty());
14090        let event = saved
14091            .events
14092            .iter()
14093            .find(|e| e.node == "prep-intent")
14094            .unwrap();
14095        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14096        assert_eq!(intent["branch"], branch);
14097        assert_eq!(intent["start"], runner.state.base_commit);
14098        assert_eq!(
14099            intent["worktree"],
14100            runner
14101                .state
14102                .worktree_root()
14103                .join(format!("cand-{label}"))
14104                .to_string_lossy()
14105                .as_ref()
14106        );
14107        assert_eq!(intent["picks"], serde_json::json!([]));
14108    }
14109
14110    #[tokio::test]
14111    async fn invalid_pinned_sources_create_no_partial_worktrees() {
14112        ask_test_home();
14113        let tmp = tempfile::tempdir().unwrap();
14114        let repo = tmp.path().join("repo");
14115        std::fs::create_dir_all(&repo).unwrap();
14116        init_repo(&repo);
14117        let mut runner = runner_at(RunStatus::Prep);
14118        runner.state.repo = repo.clone();
14119        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14120        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14121        runner.state.sources.pick_commits.push(refs::Input {
14122            token: "missing".into(),
14123            sha: "0".repeat(40),
14124            provenance: refs::Provenance::ExplicitCli,
14125        });
14126        assert!(runner.prep().await.is_err());
14127        assert!(runner.state.candidates.is_empty());
14128        assert!(!runner.state.worktree_root().exists());
14129    }
14130
14131    #[tokio::test]
14132    async fn prose_and_legacy_inference_never_import_into_candidates() {
14133        ask_test_home();
14134        let tmp = tempfile::tempdir().unwrap();
14135        let repo = tmp.path().join("repo");
14136        std::fs::create_dir_all(&repo).unwrap();
14137        init_repo(&repo);
14138        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14139        git::git(&repo, &["checkout", "-b", "magi/history/A"])
14140            .await
14141            .unwrap();
14142        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14143        git::git(&repo, &["add", "."]).await.unwrap();
14144        git::git(&repo, &["commit", "-m", "historical implementation"])
14145            .await
14146            .unwrap();
14147        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14148        git::git(&repo, &["checkout", "main"]).await.unwrap();
14149        for instruction in [
14150            format!("Example: magi/history/A at {source}"),
14151            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14152            format!("Never import magi/history/A or {source}"),
14153            format!("Conversation answer: I mentioned {source} only as background"),
14154        ] {
14155            let mut runner = runner_at(RunStatus::Prep);
14156            runner.state.repo = repo.clone();
14157            runner.state.base_commit = base.clone();
14158            runner.state.instruction = instruction.clone();
14159            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14160            runner.state.config.blind.commit_msg_hook = false;
14161            runner.state.seeds = vec![refs::Seed {
14162                token: "magi/history/A".into(),
14163                sha: source.clone(),
14164                kind: refs::SeedKind::Unmerged,
14165                branch: true,
14166                detail: String::new(),
14167            }];
14168            runner.roles.implementers = vec![spec("one")];
14169            assert_eq!(seeded_instruction(&runner.state), instruction);
14170            // Model an old record with inferred seeds but no structured input field.
14171            let mut legacy = serde_json::to_value(&runner.state).unwrap();
14172            legacy.as_object_mut().unwrap().remove("sources");
14173            runner.state = serde_json::from_value(legacy).unwrap();
14174            assert_eq!(runner.state.sources, refs::Sources::default());
14175            runner.state.save().unwrap();
14176            runner.state = RunState::load(&runner.state.id).unwrap();
14177            runner.prep().await.unwrap();
14178            for candidate in &runner.state.candidates {
14179                assert_eq!(
14180                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14181                    base
14182                );
14183                assert!(!candidate.worktree.join("unwanted").exists());
14184            }
14185        }
14186    }
14187
14188    #[tokio::test]
14189    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14190        ask_test_home();
14191        let tmp = tempfile::tempdir().unwrap();
14192        let repo = tmp.path().join("repo");
14193        std::fs::create_dir_all(&repo).unwrap();
14194        init_repo(&repo);
14195        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14196        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14197        std::fs::write(repo.join("referenced"), "required commit").unwrap();
14198        git::git(&repo, &["add", "."]).await.unwrap();
14199        git::git(&repo, &["commit", "-m", "required seed"])
14200            .await
14201            .unwrap();
14202        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14203        git::git(&repo, &["checkout", "main"]).await.unwrap();
14204        let mut runner = runner_at(RunStatus::Prep);
14205        runner.state.repo = repo.clone();
14206        runner.state.base_commit = base.clone();
14207        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14208        runner.state.config.graph.advise = false;
14209        runner.state.config.blind.commit_msg_hook = false;
14210        runner.state.sources =
14211            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14212                .await
14213                .unwrap();
14214        runner.roles.implementers = vec![spec("one"), spec("two")];
14215        runner.roles.judges = vec![spec("judge")];
14216        let label = blind::assign_labels(2, runner.state.seed)[0];
14217        let branch = runner.state.branch_for(label);
14218        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14219        git::worktree_add_branch(&repo, &worktree, &branch, &base)
14220            .await
14221            .unwrap();
14222        runner.state.candidates.push(Candidate {
14223            index: 0,
14224            label,
14225            agent: "one".into(),
14226            branch,
14227            worktree,
14228            summary: String::new(),
14229            stat: String::new(),
14230            files: 0,
14231            commits: 0,
14232            empty: false,
14233            failed: None,
14234            verified_noop: None,
14235            duration_ms: 0,
14236            folded: false,
14237        });
14238        runner.state.save().unwrap();
14239        // Reload the checkpoint immediately before its first cherry-pick.
14240        runner.state = RunState::load(&runner.state.id).unwrap();
14241        assert_eq!(runner.state.prep_complete, Some(false));
14242        runner.prep().await.unwrap();
14243        assert_eq!(runner.state.prep_complete, Some(true));
14244        assert_eq!(runner.state.candidates.len(), 2);
14245        for candidate in &runner.state.candidates {
14246            assert_eq!(
14247                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14248                git::tree_of(&repo, &seed).await.unwrap()
14249            );
14250            assert_eq!(
14251                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14252                "required commit"
14253            );
14254        }
14255        assert!(runner.state.worktree_root().join("judge-1").exists());
14256        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14257        runner.prep().await.unwrap();
14258        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14259
14260        runner.state.prep_complete = Some(false);
14261        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14262            run: Some(format!(
14263                "test -f '{}'",
14264                tmp.path().join("dependency-ready").display()
14265            )),
14266            ..Default::default()
14267        }];
14268        assert!(runner.prep().await.is_err());
14269        assert!(runner.state.prep_failure.is_some());
14270        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14271        runner.state = RunState::load(&runner.state.id).unwrap();
14272        assert!(
14273            runner.prep().await.is_err(),
14274            "correcting the cause alone must not auto-retry"
14275        );
14276        let failed_at = runner.state.events.last().unwrap().at;
14277        assert!(
14278            runner.retry_preparation(failed_at).is_err(),
14279            "old approvals cannot unlock the hold"
14280        );
14281        let mut task = crate::queue::Task::new(
14282            "retry".into(),
14283            "retry setup".into(),
14284            repo.clone(),
14285            crate::queue::Source::Human,
14286        );
14287        task.start(runner.state.id.clone());
14288        task.hold_machine(Some("setup failed".into()));
14289        task.release_operator("cli", None);
14290        runner
14291            .retry_preparation(task.resumption.as_ref().unwrap().at)
14292            .unwrap();
14293        runner.prep().await.unwrap();
14294        assert!(runner.state.prep_failure.is_none());
14295        assert_eq!(runner.state.prep_complete, Some(true));
14296        assert_eq!(runner.state.candidates.len(), 2);
14297        assert!(runner.state.worktree_root().join("judge-1").exists());
14298        for candidate in &runner.state.candidates {
14299            assert_eq!(
14300                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14301                git::tree_of(&repo, &seed).await.unwrap()
14302            );
14303        }
14304    }
14305
14306    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14307        let mut heads = Vec::new();
14308        for candidate in candidates {
14309            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14310        }
14311        heads
14312    }
14313
14314    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14315        ask_test_home();
14316        let tmp = tempfile::tempdir().unwrap();
14317        let repo = tmp.path().join("repo");
14318        std::fs::create_dir(&repo).unwrap();
14319        init_repo(&repo);
14320        let mut runner = runner_at(RunStatus::Reviewing);
14321        runner.state.repo = repo.clone();
14322        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14323        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14324        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14325            let branch = format!("candidate-{label}");
14326            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14327            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14328                .await
14329                .unwrap();
14330            if label == 'B' {
14331                std::fs::write(worktree.join("change"), "winner").unwrap();
14332                git::rescue_commit(&worktree, "candidate change")
14333                    .await
14334                    .unwrap();
14335            }
14336            runner.state.candidates.push(Candidate {
14337                index,
14338                label,
14339                agent: "alpha".into(),
14340                branch,
14341                worktree,
14342                summary: "completed reply".into(),
14343                stat: String::new(),
14344                files: usize::from(label == 'B'),
14345                commits: usize::from(label == 'B'),
14346                empty: label == 'A',
14347                failed: None,
14348                verified_noop: None,
14349                duration_ms: 1,
14350                folded: false,
14351            });
14352        }
14353        runner.state.judge_skipped = true;
14354        runner.state.tally = Some(Tally {
14355            first_choice: BTreeMap::new(),
14356            borda: BTreeMap::new(),
14357            winner: 'B',
14358            rankings: 0,
14359            unanimous_initial: false,
14360            deliberated: false,
14361            changed_votes: 0,
14362            unanimous_final: false,
14363            tie_break: None,
14364            judges: 0,
14365            present: 0,
14366            quorum: 0,
14367            met_quorum: true,
14368            uncontested: Some("only B produced a change".into()),
14369        });
14370        runner.state.save().unwrap();
14371        (tmp, runner)
14372    }
14373
14374    #[tokio::test]
14375    #[cfg(unix)]
14376    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14377        for (solo, failed_checkpoint) in
14378            [(true, false), (false, false), (true, true), (false, true)]
14379        {
14380            let (_tmp, mut runner) = preservation_resume_fixture().await;
14381            if solo {
14382                runner.state.candidates.retain(|c| c.label == 'A');
14383            }
14384            runner.state.candidates[0].empty = false;
14385            runner.state.candidates[0].summary.clear();
14386            runner.state.tally = None;
14387            runner.state.judge_skipped = false;
14388            runner.state.status = RunStatus::Implementing;
14389            runner.state.config.disk.min_free_bytes = u64::MAX;
14390            let mut implementer = spec("alpha");
14391            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14392            runner.roles.implementers = vec![implementer];
14393            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14394            if failed_checkpoint {
14395                std::fs::create_dir_all(&patch).unwrap();
14396            }
14397            runner.state.save().unwrap();
14398            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14399            runner.state = RunState::load(&runner.state.id).unwrap();
14400            assert_eq!(runner.state.status, RunStatus::Blocked);
14401            assert!(runner.state.interrupted_implementations.contains(&'A'));
14402            assert_eq!(runner.state.candidates[0].commits, 0);
14403            assert!(!runner.state.candidates[0].empty);
14404            assert!(runner.state.candidates[0].failed.is_none());
14405            if failed_checkpoint {
14406                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14407                std::fs::remove_dir(patch).unwrap();
14408            }
14409            runner.state.resource_block = None;
14410            runner.state.config.disk.min_free_bytes = 0;
14411            runner.roles.implementers[0].command = vec![
14412                "sh".into(),
14413                "-c".into(),
14414                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14415            ];
14416            runner.implement().await.unwrap();
14417            assert!(runner.state.interrupted_implementations.is_empty());
14418            assert!(runner.state.candidate_preservation.is_empty());
14419            assert_eq!(runner.state.candidates[0].commits, 1);
14420            assert!(!runner.state.candidates[0].empty);
14421            assert!(
14422                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14423                    .unwrap()
14424                    .contains("retried empty interrupted seat")
14425            );
14426            if !solo {
14427                assert_eq!(runner.state.candidates[1].commits, 1);
14428            }
14429        }
14430    }
14431
14432    #[tokio::test]
14433    #[cfg(unix)]
14434    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14435        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14436            let (_tmp, mut runner) = preservation_resume_fixture().await;
14437            runner.state.candidates.retain(|c| c.label == 'A');
14438            runner.state.candidates[0].empty = false;
14439            runner.state.candidates[0].summary.clear();
14440            runner.state.tally = None;
14441            runner.state.judge_skipped = false;
14442            runner.state.status = RunStatus::Implementing;
14443            // Force the real post-spawn watchdog, independently of available
14444            // disk space. Dirty/committed work predates its first measurement.
14445            runner.state.config.disk.min_free_bytes = u64::MAX;
14446            let wt = runner.state.candidates[0].worktree.clone();
14447            std::fs::write(
14448                wt.join("interrupted-source"),
14449                "retain implementation work\n",
14450            )
14451            .unwrap();
14452            if committed {
14453                git::rescue_commit(&wt, "implementation checkpoint")
14454                    .await
14455                    .unwrap();
14456            }
14457            let lock = git::git(
14458                &wt,
14459                &[
14460                    "rev-parse",
14461                    "--path-format=absolute",
14462                    "--git-path",
14463                    "index.lock",
14464                ],
14465            )
14466            .await
14467            .unwrap();
14468            if block_rescue {
14469                std::fs::write(&lock, "fixture-owned lock").unwrap();
14470            }
14471            let mut implementer = spec("alpha");
14472            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14473            runner.roles.implementers = vec![implementer];
14474            runner.state.save().unwrap();
14475            let result = runner.implement().await;
14476            assert_eq!(result.is_err(), block_rescue);
14477            assert_eq!(
14478                runner.state.resource_block.as_ref().unwrap().kind,
14479                "disk pressure"
14480            );
14481            assert!(
14482                runner
14483                    .state
14484                    .events
14485                    .iter()
14486                    .any(|e| e.message.contains("agent execution interrupted"))
14487            );
14488            assert!(
14489                runner
14490                    .state
14491                    .dir()
14492                    .join("artifacts/candidate-A-recovery/staged.patch")
14493                    .exists()
14494            );
14495            if block_rescue {
14496                let saved = RunState::load(&runner.state.id).unwrap();
14497                assert!(saved.candidate_preservation.contains_key(&'A'));
14498                assert!(!saved.candidates[0].empty);
14499                assert_eq!(
14500                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14501                    "retain implementation work\n"
14502                );
14503                std::fs::remove_file(lock).unwrap();
14504                runner.state = saved;
14505            } else {
14506                assert_eq!(runner.state.status, RunStatus::Blocked);
14507                assert_eq!(runner.state.candidates[0].commits, 1);
14508            }
14509            // Recovery must save the candidate before deciding whether another
14510            // implementation is needed. No implementer is available on retry.
14511            runner.roles.implementers.clear();
14512            runner.state.config.disk.min_free_bytes = 0;
14513            runner.state.resource_block = None;
14514            runner.implement().await.unwrap();
14515            assert!(runner.state.candidate_preservation.is_empty());
14516            assert_eq!(runner.state.candidates[0].commits, 1);
14517            assert!(!runner.state.candidates[0].empty);
14518            assert!(git::is_clean(&wt).await.unwrap());
14519            assert!(
14520                crate::run::read_artifact(&runner.state, "cand-A.patch")
14521                    .unwrap()
14522                    .contains("retain implementation work")
14523            );
14524        }
14525    }
14526
14527    #[tokio::test]
14528    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14529        let (_tmp, mut runner) = preservation_resume_fixture().await;
14530        // Schema 15 already confirmed the unfolded no-change result.
14531        runner.implement().await.unwrap();
14532        assert!(runner.state.candidate_preservation.is_empty());
14533        assert!(
14534            !runner
14535                .state
14536                .dir()
14537                .join("artifacts/candidate-A-recovery")
14538                .exists()
14539        );
14540        runner.fold_losers().await.unwrap();
14541        assert!(!runner.state.candidates[0].worktree.exists());
14542        for schema in [14, crate::run::SCHEMA] {
14543            runner.state.schema = schema;
14544            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14545            runner.state.save().unwrap();
14546            runner.state = RunState::load(&runner.state.id).unwrap();
14547            runner.implement().await.unwrap();
14548            assert!(runner.state.candidate_preservation.is_empty());
14549            assert!(runner.state.candidates[0].folded);
14550            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14551        }
14552    }
14553
14554    #[tokio::test]
14555    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14556        let (_tmp, mut runner) = preservation_resume_fixture().await;
14557        runner.state.candidates.retain(|c| c.label == 'A');
14558        runner.state.tally = None;
14559        runner.state.judge_skipped = false;
14560        let candidate = &mut runner.state.candidates[0];
14561        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14562        candidate.failed = Some("deployment validation unavailable".into());
14563        std::fs::write(
14564            candidate.worktree.join("repair"),
14565            "future launcher policy\n",
14566        )
14567        .unwrap();
14568        runner
14569            .state
14570            .candidate_preservation
14571            .insert('A', "awaiting persistence".into());
14572        runner.persist_candidates().await.unwrap();
14573        let candidate = &runner.state.candidates[0];
14574        assert!(!candidate.empty);
14575        assert_eq!(candidate.commits, 1);
14576        assert_eq!(candidate.files, 1);
14577        assert!(candidate.failed.is_none());
14578        assert!(candidate.summary.contains("EPERM"));
14579        assert!(
14580            crate::run::read_artifact(&runner.state, "cand-A.patch")
14581                .unwrap()
14582                .contains("future launcher policy")
14583        );
14584        assert!(runner.state.candidate_preservation.is_empty());
14585    }
14586
14587    #[tokio::test]
14588    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14589        let (_tmp, mut runner) = preservation_resume_fixture().await;
14590        let wt = runner.state.candidates[0].worktree.clone();
14591        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14592        runner.state.schema = 14;
14593        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14594        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14595        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14596        for _ in 0..2 {
14597            assert!(runner.implement().await.is_err());
14598            assert_eq!(runner.state.status, RunStatus::Blocked);
14599            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14600            assert!(
14601                runner.state.candidates[0].empty,
14602                "old verdict must remain unusable"
14603            );
14604            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14605            assert!(runner.state.judge_skipped);
14606            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14607            assert!(
14608                crate::run::read_artifact(&runner.state, "cand-A.patch")
14609                    .unwrap()
14610                    .contains("owned edits")
14611            );
14612            assert!(wt.exists());
14613            assert_eq!(
14614                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14615                    .await
14616                    .unwrap(),
14617                1
14618            );
14619            runner.state = RunState::load(&runner.state.id).unwrap();
14620        }
14621    }
14622
14623    #[tokio::test]
14624    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14625        let (_tmp, mut runner) = preservation_resume_fixture().await;
14626        let wt = runner.state.candidates[0].worktree.clone();
14627        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14628        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14629        let index_lock = PathBuf::from(
14630            git::git(
14631                &wt,
14632                &[
14633                    "rev-parse",
14634                    "--path-format=absolute",
14635                    "--git-path",
14636                    "index.lock",
14637                ],
14638            )
14639            .await
14640            .unwrap(),
14641        );
14642        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14643        runner.state.schema = 14;
14644        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14645        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14646        assert!(runner.implement().await.is_err());
14647        assert!(runner.after_implement().is_err());
14648        assert!(runner.fold_losers().await.is_err());
14649        assert_eq!(
14650            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14651            "failed seat edits"
14652        );
14653        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14654        runner.state = RunState::load(&runner.state.id).unwrap();
14655        std::fs::remove_file(index_lock).unwrap();
14656        // Recovery finds changes that the old uncontested result never judged.
14657        assert!(runner.implement().await.is_err());
14658        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14659        assert!(runner.state.candidates[0].empty);
14660        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14661        assert!(
14662            crate::run::read_artifact(&runner.state, "cand-A.patch")
14663                .unwrap()
14664                .contains("failed seat edits")
14665        );
14666        assert!(wt.exists());
14667    }
14668
14669    #[tokio::test]
14670    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14671        let (_tmp, mut runner) = preservation_resume_fixture().await;
14672        let repo = runner.state.repo.clone();
14673        let common = git::git(
14674            &repo,
14675            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14676        )
14677        .await
14678        .unwrap();
14679        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14680        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14681        assert!(runner.fold_losers().await.is_err());
14682        assert!(!runner.state.candidates[0].worktree.exists());
14683        assert!(!runner.state.candidates[0].folded);
14684        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14685        runner.state = RunState::load(&runner.state.id).unwrap();
14686        std::fs::remove_file(branch_lock).unwrap();
14687        let judge = runner.state.worktree_root().join("judge-1");
14688        git::worktree_add_detached(&repo, &judge, "main")
14689            .await
14690            .unwrap();
14691        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14692        runner.roles.judges = vec![spec("judge")];
14693        assert!(runner.fold_losers().await.is_err());
14694        runner.state = RunState::load(&runner.state.id).unwrap();
14695        assert!(
14696            runner.state.candidates[0].folded,
14697            "candidate progress saved before judge failure"
14698        );
14699        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14700        assert!(judge.join("uncommitted").exists());
14701        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14702        runner.fold_losers().await.unwrap();
14703        assert!(!judge.exists());
14704        runner.fold_losers().await.unwrap();
14705    }
14706
14707    #[tokio::test]
14708    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14709        let (_tmp, mut runner) = preservation_resume_fixture().await;
14710        runner.state.candidates.retain(|c| c.label == 'B');
14711        runner.state.tally = None;
14712        runner.state.judge_skipped = false;
14713        runner.state.prep_complete = Some(true);
14714        runner.state.config.graph.advise = false;
14715        runner.state.config.graph.selected_implementer = Some("alpha".into());
14716        let wt = runner.state.candidates[0].worktree.clone();
14717        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14718        runner
14719            .state
14720            .candidate_preservation
14721            .insert('B', "awaiting persistence".into());
14722        let index_lock = git::git(
14723            &wt,
14724            &[
14725                "rev-parse",
14726                "--path-format=absolute",
14727                "--git-path",
14728                "index.lock",
14729            ],
14730        )
14731        .await
14732        .unwrap();
14733        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14734        runner.state.save().unwrap();
14735        assert!(runner.execute_graph().await.is_err());
14736        assert_eq!(runner.state.status, RunStatus::Blocked);
14737        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14738        std::fs::remove_file(index_lock).unwrap();
14739        // Park at the next graph boundary: this tests the real selected-seat
14740        // hold guard without dispatching reviewers or publishing anything.
14741        for _ in 0..2 {
14742            runner.state = RunState::load(&runner.state.id).unwrap();
14743            runner.pause.park();
14744            runner.execute_graph().await.unwrap();
14745            assert_eq!(runner.state.status, RunStatus::Implementing);
14746            assert!(runner.state.candidate_preservation.is_empty());
14747            assert_eq!(runner.state.candidates[0].commits, 2);
14748            assert!(runner.state.tally.is_none());
14749        }
14750        runner.implement().await.unwrap();
14751        assert_eq!(runner.state.status, RunStatus::Judging);
14752        // A genuine selected-implementer hold must still require release.
14753        runner.state.status = RunStatus::Blocked;
14754        runner.state.candidates[0].failed = Some("implementation failed".into());
14755        runner.state.save().unwrap();
14756        let error = runner.execute_graph().await.unwrap_err();
14757        assert!(
14758            error
14759                .to_string()
14760                .contains("selected implementation is held")
14761        );
14762    }
14763
14764    #[tokio::test]
14765    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14766        let (_tmp, mut runner) = preservation_resume_fixture().await;
14767        let repo = runner.state.repo.clone();
14768        let candidate = runner.state.candidates[0].clone();
14769        // Remove only the disposable fixture directory, leaving Git's linked
14770        // worktree registration and branch intact, as in a vanished checkout.
14771        let registered = git::worktree_holding(&repo, &candidate.branch)
14772            .await
14773            .unwrap()
14774            .unwrap();
14775        assert_eq!(
14776            registered.canonicalize().unwrap(),
14777            candidate.worktree.canonicalize().unwrap()
14778        );
14779        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14780        assert_eq!(
14781            git::worktree_holding(&repo, &candidate.branch)
14782                .await
14783                .unwrap(),
14784            Some(registered)
14785        );
14786        assert!(
14787            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14788            "the stale registration initially keeps the branch checked out"
14789        );
14790        runner.fold_losers().await.unwrap();
14791        assert!(runner.state.candidates[0].folded);
14792        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14793        assert_eq!(
14794            git::worktree_holding(&repo, &candidate.branch)
14795                .await
14796                .unwrap(),
14797            None
14798        );
14799        runner.state = RunState::load(&runner.state.id).unwrap();
14800        runner.fold_losers().await.unwrap();
14801    }
14802
14803    #[tokio::test]
14804    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14805        let (_tmp, mut runner) = preservation_resume_fixture().await;
14806        let repo = runner.state.repo.clone();
14807        let candidate = runner.state.candidates[0].clone();
14808        let winner = runner.state.candidates[1].clone();
14809        let registered = git::worktree_holding(&repo, &candidate.branch)
14810            .await
14811            .unwrap()
14812            .unwrap();
14813        assert_eq!(
14814            registered.canonicalize().unwrap(),
14815            candidate.worktree.canonicalize().unwrap()
14816        );
14817        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14818        assert_eq!(
14819            git::worktree_holding(&repo, &candidate.branch)
14820                .await
14821                .unwrap(),
14822            Some(registered)
14823        );
14824        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14825
14826        for _ in 0..2 {
14827            fold_run(&mut runner.state, false, &crate::run::home())
14828                .await
14829                .unwrap();
14830            runner.state = RunState::load(&runner.state.id).unwrap();
14831            assert!(runner.state.candidates[0].folded);
14832            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14833            assert_eq!(
14834                git::worktree_holding(&repo, &candidate.branch)
14835                    .await
14836                    .unwrap(),
14837                None
14838            );
14839            assert!(!runner.state.candidates[1].folded);
14840            assert!(winner.worktree.exists());
14841            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14842        }
14843    }
14844
14845    #[tokio::test]
14846    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14847        ask_test_home();
14848        let tmp = tempfile::tempdir().unwrap();
14849        let repo = tmp.path().join("repo");
14850        std::fs::create_dir(&repo).unwrap();
14851        init_repo(&repo);
14852        let mut runner = runner_at(RunStatus::Implementing);
14853        runner.state.repo = repo.clone();
14854        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14855        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14856        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14857            let branch = format!("candidate-{label}");
14858            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14859            git::worktree_add_branch(&repo, &wt, &branch, "main")
14860                .await
14861                .unwrap();
14862            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14863            git::git(&wt, &["add", "README.md"]).await.unwrap();
14864            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14865            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14866            runner.state.candidates.push(Candidate {
14867                index,
14868                label,
14869                agent: "alpha".to_owned(),
14870                branch,
14871                worktree: wt,
14872                summary: "completed implementation".to_owned(),
14873                stat: String::new(),
14874                files: 0,
14875                commits: 0,
14876                empty: false,
14877                failed: None,
14878                verified_noop: None,
14879                duration_ms: 42,
14880                folded: false,
14881            });
14882            runner
14883                .state
14884                .candidate_preservation
14885                .insert(label, "awaiting persistence".to_owned());
14886        }
14887        let wt = runner.state.candidates[0].worktree.clone();
14888        let index_lock = git::git(
14889            &wt,
14890            &[
14891                "rev-parse",
14892                "--path-format=absolute",
14893                "--git-path",
14894                "index.lock",
14895            ],
14896        )
14897        .await
14898        .unwrap();
14899        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14900        runner.state.save().unwrap();
14901        assert!(runner.persist_candidates().await.is_err());
14902        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14903        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14904        assert_eq!(
14905            runner.state.candidates[1].commits, 1,
14906            "other results are still collected"
14907        );
14908        assert!(
14909            !runner.state.candidates[0].empty,
14910            "failed rescue is not no change"
14911        );
14912        assert_eq!(
14913            runner.state.candidates[0].summary,
14914            "completed implementation"
14915        );
14916        assert!(runner.after_implement().is_err());
14917        let home = tmp.path().join("fold-home");
14918        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14919        assert!(wt.join("new-file").exists());
14920        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14921        assert!(recovery.join("staged.patch").exists());
14922        assert!(recovery.join("unstaged.patch").exists());
14923        assert_eq!(
14924            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14925            "untracked edits\n"
14926        );
14927        runner.state.begin_pass();
14928        runner.state.end_pass_parked(ParkCause::Upgrade {
14929            from: "1.0".into(),
14930            to: Some("1.1".into()),
14931        });
14932        runner.state.save().unwrap();
14933        runner.state = RunState::load(&runner.state.id).unwrap();
14934        assert!(matches!(
14935            runner.state.passes.last().unwrap().end,
14936            Some(crate::run::PassEnd::Parked {
14937                cause: ParkCause::Upgrade { .. }
14938            })
14939        ));
14940        std::fs::remove_file(index_lock).unwrap();
14941        // No implementer is configured: a persistence retry cannot dispatch a seat.
14942        runner.implement().await.unwrap();
14943        assert!(runner.state.candidate_preservation.is_empty());
14944        assert!(
14945            runner
14946                .state
14947                .candidates
14948                .iter()
14949                .all(|c| c.commits == 1 && !c.empty)
14950        );
14951        assert!(git::is_clean(&wt).await.unwrap());
14952        assert!(
14953            crate::run::read_artifact(&runner.state, "cand-A.patch")
14954                .unwrap()
14955                .contains("later tracked edits")
14956        );
14957    }
14958
14959    #[tokio::test]
14960    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14961        ask_test_home();
14962        let tmp = tempfile::tempdir().unwrap();
14963        let repo = tmp.path().join("repo");
14964        std::fs::create_dir(&repo).unwrap();
14965        init_repo(&repo);
14966        let wt = tmp.path().join("candidate");
14967        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14968            .await
14969            .unwrap();
14970        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14971        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14972        assert!(wt.join("uncommitted").exists());
14973        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14974        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14975        let mut runner = runner_at(RunStatus::Implementing);
14976        runner.state.repo = repo.clone();
14977        runner.state.base_commit = "missing-base".to_owned();
14978        runner.state.candidates.push(Candidate {
14979            index: 0,
14980            label: 'A',
14981            agent: "alpha".into(),
14982            branch: "candidate".into(),
14983            worktree: wt.clone(),
14984            summary: "completed".into(),
14985            stat: String::new(),
14986            files: 0,
14987            commits: 0,
14988            empty: false,
14989            failed: None,
14990            verified_noop: None,
14991            duration_ms: 1,
14992            folded: false,
14993        });
14994        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14995        assert!(
14996            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14997                .await
14998                .is_err()
14999        );
15000        assert!(!runner.state.candidates[0].folded);
15001        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15002        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15003        runner
15004            .state
15005            .candidate_preservation
15006            .insert('A', "pending".into());
15007        assert!(runner.persist_candidates().await.is_err());
15008        assert!(!runner.state.candidates[0].empty);
15009        assert!(!runner.state.candidates[0].folded);
15010        assert!(wt.exists());
15011        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15012        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15013        std::fs::create_dir_all(&patch_path).unwrap();
15014        assert!(
15015            runner.persist_candidates().await.is_err(),
15016            "artifact failure stays pending"
15017        );
15018        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15019        assert!(!runner.state.candidates[0].empty);
15020        std::fs::remove_dir(patch_path).unwrap();
15021        runner.persist_candidates().await.unwrap();
15022        assert!(
15023            runner.state.candidates[0].empty,
15024            "only successful inspection confirms no change"
15025        );
15026    }
15027
15028    fn init_repo(dir: &Path) {
15029        let run = |args: &[&str]| {
15030            let out = std::process::Command::new("git")
15031                .args(args)
15032                .current_dir(dir)
15033                .quiet()
15034                .output()
15035                .expect("spawn git");
15036            assert!(
15037                out.status.success(),
15038                "git {args:?} failed: {}",
15039                String::from_utf8_lossy(&out.stderr)
15040            );
15041        };
15042        run(&["init", "-b", "main"]);
15043        run(&["config", "user.name", "magi test"]);
15044        run(&["config", "user.email", "magi@example.com"]);
15045        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15046        run(&["add", "-A"]);
15047        run(&["commit", "-m", "init"]);
15048    }
15049
15050    // `settle_questions` is what closes the ghost the phone showed: a run's
15051    // seat asked something, the run then ended, and nothing was left to
15052    // abandon the question it left `open`. `HOME` is a process-wide
15053    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
15054    // first time it runs in the binary — every test below still reaches the
15055    // same directory whichever call won, and each gets its own run id from
15056    // `RunState::new`, so they never collide there.
15057    fn ask_test_home() {
15058        crate::run::pin_test_home();
15059    }
15060
15061    #[cfg(unix)]
15062    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15063        ask_test_home();
15064        let tmp = tempfile::tempdir().unwrap();
15065        let repo = tmp.path().join("repo");
15066        std::fs::create_dir(&repo).unwrap();
15067        init_repo(&repo);
15068        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15069        let origin = tmp.path().join("origin.git");
15070        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15071            .await
15072            .unwrap();
15073        git::git(
15074            &repo,
15075            &["remote", "add", "origin", origin.to_str().unwrap()],
15076        )
15077        .await
15078        .unwrap();
15079        git::git(&repo, &["checkout", "-b", "fix/retained"])
15080            .await
15081            .unwrap();
15082        std::fs::write(repo.join("retained"), "retained change").unwrap();
15083        git::git(&repo, &["add", "."]).await.unwrap();
15084        git::git(&repo, &["commit", "-m", "retained implementation"])
15085            .await
15086            .unwrap();
15087        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15088        git::git(&repo, &["checkout", "main"]).await.unwrap();
15089        std::fs::write(repo.join("base-change"), "new base").unwrap();
15090        git::git(&repo, &["add", "."]).await.unwrap();
15091        git::git(&repo, &["commit", "-m", "base moved"])
15092            .await
15093            .unwrap();
15094        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15095        git::git(&repo, &["checkout", "fix/retained"])
15096            .await
15097            .unwrap();
15098        let mut runner = runner_at(RunStatus::Voting);
15099        runner.state.repo = repo.clone();
15100        runner.state.base_commit = base;
15101        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15102        runner.state.config.disk.min_free_bytes = 0;
15103        runner.state.config.graph.review_rounds = 1;
15104        runner.state.config.verify.e2e = vec!["true".into()];
15105        runner.state.config.verify.gate = vec!["true".into()];
15106        runner.state.candidates = vec![Candidate {
15107            index: 0,
15108            label: 'A',
15109            agent: "fixture".into(),
15110            branch: "fix/retained".into(),
15111            worktree: repo.clone(),
15112            summary: String::new(),
15113            stat: String::new(),
15114            files: 1,
15115            commits: 1,
15116            empty: false,
15117            failed: None,
15118            verified_noop: None,
15119            duration_ms: 0,
15120            folded: false,
15121        }];
15122        runner.state.tally = Some(Tally {
15123            first_choice: BTreeMap::new(),
15124            borda: BTreeMap::new(),
15125            winner: 'A',
15126            rankings: 0,
15127            unanimous_initial: true,
15128            deliberated: false,
15129            changed_votes: 0,
15130            unanimous_final: true,
15131            tie_break: None,
15132            judges: 0,
15133            present: 0,
15134            quorum: 0,
15135            met_quorum: true,
15136            uncontested: Some("review-only".into()),
15137        });
15138        let mut task = crate::queue::Task::new(
15139            "fixture".into(),
15140            "fixture".into(),
15141            repo,
15142            crate::queue::Source::Human,
15143        );
15144        task.link_run(&runner.state.id);
15145        crate::queue::Queue::open().put(&mut task).unwrap();
15146        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15147        runner.state.base_sync = Some(BaseSync {
15148            tip: String::new(),
15149            behind: 1,
15150            attempts: BASE_SYNC_ROUNDS,
15151            conflict: None,
15152            already_in: None,
15153        });
15154        runner.sync_to_base().await.unwrap();
15155        assert_eq!(runner.state.status, RunStatus::Blocked);
15156        (tmp, runner, head, task.id)
15157    }
15158
15159    #[cfg(unix)]
15160    #[tokio::test]
15161    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15162        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15163        if std::env::var_os(CHILD).is_none() {
15164            let home = tempfile::tempdir().unwrap();
15165            let output = std::process::Command::new(std::env::current_exe().unwrap())
15166                .args([
15167                    "--exact",
15168                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15169                    "--nocapture",
15170                ])
15171                .env(CHILD, "1")
15172                .env("MAGI_HOME", home.path())
15173                .env("MAGI_WORKER_HOME", home.path())
15174                .env("MAGI_WORKER_ID", "abcdef")
15175                .output()
15176                .unwrap();
15177            assert!(
15178                output.status.success()
15179                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15180                "{}\n{}",
15181                String::from_utf8_lossy(&output.stdout),
15182                String::from_utf8_lossy(&output.stderr)
15183            );
15184            return;
15185        }
15186        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15187        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15188        let home = crate::run::home();
15189        let queue = crate::queue::Queue::open();
15190        let mut owner = queue.get(&task).unwrap();
15191        owner.release_operator("test-release", None);
15192        owner.start(runner.state.id.clone());
15193        queue.put(&mut owner).unwrap();
15194        runner.state.driver_exited = true;
15195        runner.state.task_instruction = Some(owner.instruction.clone());
15196        runner.state.save().unwrap();
15197        let _claim = queue.claim(&task).unwrap();
15198        let reserve = |id: &str| {
15199            let directory = home.join("workers/jobs").join(id);
15200            std::fs::create_dir_all(&directory).unwrap();
15201            let job = crate::worker::Job {
15202                id: id.into(),
15203                sequence: 1,
15204                kind: "task".into(),
15205                key: format!("task:{task}"),
15206                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15207                generation: "fixture-generation".into(),
15208                version: None,
15209                status: crate::worker::Status::Running,
15210                result: None,
15211                pid: Some(std::process::id()),
15212                created_at: 0,
15213                finished_at: None,
15214            };
15215            std::fs::write(
15216                directory.join("job.json"),
15217                serde_json::to_vec(&job).unwrap(),
15218            )
15219            .unwrap();
15220            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15221        };
15222        let _own = reserve("abcdef");
15223        let other = reserve("fedcba");
15224        assert!(
15225            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15226                .await
15227                .is_err()
15228        );
15229        assert!(
15230            RunState::load(&runner.state.id)
15231                .unwrap()
15232                .base_sync_recovery
15233                .authorizations
15234                .is_empty()
15235        );
15236        drop(other);
15237        assert!(crate::daemon::is_working_on(
15238            &home,
15239            &runner.state.id,
15240            Timestamp::now()
15241        ));
15242        assert!(
15243            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15244        );
15245        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15246            .await
15247            .unwrap();
15248        assert!(
15249            RunState::load(&runner.state.id)
15250                .unwrap()
15251                .base_recovery_pending()
15252        );
15253    }
15254
15255    #[cfg(unix)]
15256    #[tokio::test]
15257    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15258        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15259        let home = crate::run::home();
15260        runner.state.driver_pid = Some(std::process::id());
15261        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15262        runner.state.driver_exited = false;
15263        runner.state.begin_pass();
15264        runner.state.save().unwrap();
15265        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15266        driver.record(&runner.state).unwrap();
15267        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15268        // Preserve the durable claim as if the process died before Drop saved.
15269        drop(driver);
15270        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15271        assert_eq!(
15272            runner.state.liveness_under(&home, false),
15273            crate::run::Liveness::Dead
15274        );
15275        let queue = crate::queue::Queue::open();
15276        let _claim = queue.claim(&task).unwrap();
15277        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15278            .await
15279            .unwrap();
15280        let disk = RunState::load(&runner.state.id).unwrap();
15281        assert!(!disk.driver_exited);
15282        assert!(disk.base_recovery_pending());
15283    }
15284
15285    #[cfg(unix)]
15286    #[tokio::test]
15287    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15288        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15289        if std::env::var_os(CHILD).is_none() {
15290            use std::os::unix::fs::PermissionsExt;
15291            let tmp = tempfile::tempdir().unwrap();
15292            let gh = tmp.path().join("gh");
15293            std::fs::write(&gh, r#"#!/bin/sh
15294case "$1 $2" in
15295  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15296  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15297  *) exit 2 ;;
15298esac
15299"#).unwrap();
15300            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15301            let mut paths = vec![tmp.path().to_path_buf()];
15302            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15303            let output = std::process::Command::new(std::env::current_exe().unwrap())
15304                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15305                .env(CHILD, "1")
15306                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15307                .env("PATH", std::env::join_paths(paths).unwrap())
15308                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15309                .output().unwrap();
15310            assert!(
15311                output.status.success()
15312                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15313                "{}\n{}",
15314                String::from_utf8_lossy(&output.stdout),
15315                String::from_utf8_lossy(&output.stderr)
15316            );
15317            return;
15318        }
15319        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15320        let repo = runner.state.repo.clone();
15321        git::git(&repo, &["checkout", "main"]).await.unwrap();
15322        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15323        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15324        git::git(&repo, &["checkout", "fix/retained"])
15325            .await
15326            .unwrap();
15327        runner.state.pr = Some(crate::run::PrRecord {
15328            url: "https://example.test/project/pull/1".into(),
15329            number: 1,
15330            state: "open".into(),
15331            checks: "unknown".into(),
15332            round: 0,
15333            rounds: 1,
15334            red_at_merge: Vec::new(),
15335        });
15336        runner.sync_to_base().await.unwrap();
15337        assert_eq!(runner.state.status, RunStatus::Blocked);
15338        assert_eq!(
15339            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15340            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15341        );
15342        assert!(runner.state.base_sync_closure_pending());
15343        assert!(crate::handover::can_resume(&runner.state));
15344        let queue = crate::queue::Queue::open();
15345        assert!(!crate::base_sync_recovery::held_task(
15346            &queue,
15347            &queue.get(&task).unwrap()
15348        ));
15349        assert!(runner.state.authorize_base_sync(&task).is_err());
15350        // Legacy closure stops have no typed stop but must retain the checkpoint.
15351        runner.state.base_sync_recovery.stop = None;
15352        runner.state.status = RunStatus::Voting;
15353        runner.state.gate_ran = true;
15354        runner.state.config.graph.review_rounds = 0;
15355        runner.state.save().unwrap();
15356        assert!(crate::handover::can_resume(&runner.state));
15357        runner.state = RunState::load(&runner.state.id).unwrap();
15358        runner.execute_graph().await.unwrap();
15359        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15360        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15361        assert_eq!(
15362            runner.state.base_sync.as_ref().unwrap().attempts,
15363            BASE_SYNC_ROUNDS
15364        );
15365        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15366        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15367        assert!(runner.state.reviews.is_empty());
15368        assert!(runner.state.gate.is_empty());
15369    }
15370
15371    #[cfg(unix)]
15372    #[tokio::test]
15373    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15374        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15375        runner.state.driver_exited = true;
15376        runner.state.save().unwrap();
15377        let home = crate::run::home();
15378        let queue = crate::queue::Queue::open();
15379        let _claim = queue.claim(&task).unwrap();
15380        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15381        assert!(
15382            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15383                .await
15384                .is_err()
15385        );
15386        drop(lease);
15387        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15388        assert!(
15389            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15390                .await
15391                .is_err()
15392        );
15393        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15394        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15395            .await
15396            .unwrap();
15397        let disk = RunState::load(&runner.state.id).unwrap();
15398        assert!(disk.base_recovery_pending());
15399        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15400        assert!(
15401            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15402                .await
15403                .is_err()
15404        );
15405        assert_eq!(
15406            RunState::load(&runner.state.id)
15407                .unwrap()
15408                .base_sync_recovery
15409                .authorizations
15410                .len(),
15411            1
15412        );
15413        let mut owner = queue.get(&task).unwrap();
15414        owner.runs.push("20260101-000000-ffff".into());
15415        queue.put(&mut owner).unwrap();
15416        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15417            .await
15418            .unwrap_err();
15419        assert!(error.to_string().contains("run task ownership changed"));
15420    }
15421
15422    #[cfg(unix)]
15423    #[tokio::test]
15424    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15425        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15426        let mut legacy = serde_json::to_value(&runner.state).unwrap();
15427        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15428        legacy["status"] = serde_json::json!("voting");
15429        runner.state = serde_json::from_value(legacy).unwrap();
15430        runner.state.save().unwrap();
15431        for _ in 0..3 {
15432            runner.execute_graph().await.unwrap();
15433            assert_eq!(runner.state.status, RunStatus::Blocked);
15434            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15435            assert!(!crate::handover::can_resume(&runner.state));
15436            assert_eq!(
15437                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15438                head
15439            );
15440            assert!(runner.state.reviews.is_empty());
15441            assert!(!runner.state.gate_ran);
15442        }
15443        let notice = crate::notices::run_ended(&runner.state).unwrap();
15444        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15445        runner.state.begin_pass();
15446        runner.state.status = RunStatus::Voting;
15447        runner.deliberate().await.unwrap();
15448        runner.vote().await.unwrap();
15449        runner.finish_after_tally().await.unwrap();
15450        assert_eq!(runner.state.status, RunStatus::Blocked);
15451        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15452        assert_eq!(
15453            crate::notices::run_ended(&runner.state).unwrap().since,
15454            notice.since
15455        );
15456    }
15457
15458    #[cfg(unix)]
15459    #[tokio::test]
15460    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15461        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15462        for detail in [
15463            "fix/retained conflicts with origin/main and did not rebase: conflict",
15464            "rebased fix/retained locally but origin refused the push",
15465            "unknown failure",
15466        ] {
15467            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15468            runner.state.record_base_sync_stop(&head);
15469            assert!(runner.state.authorize_base_sync(&task).is_err());
15470            runner.state.status = RunStatus::Voting;
15471            runner.execute_graph().await.unwrap();
15472            assert_eq!(runner.state.status, RunStatus::Blocked);
15473            assert!(!crate::handover::can_resume(&runner.state));
15474            assert_eq!(
15475                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15476                head
15477            );
15478        }
15479    }
15480
15481    #[cfg(unix)]
15482    #[tokio::test]
15483    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15484        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15485        runner.state.authorize_base_sync(&task).unwrap();
15486        assert!(runner.state.consume_base_recovery(&head));
15487        runner.state.save().unwrap();
15488        runner.state = RunState::load(&runner.state.id).unwrap();
15489        runner.execute_graph().await.unwrap();
15490        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15491        assert_eq!(runner.state.status, RunStatus::Blocked);
15492        assert!(!runner.state.consume_base_recovery(&head));
15493        assert!(!crate::handover::can_resume(&runner.state));
15494        runner.state.authorize_base_sync(&task).unwrap();
15495        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15496        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15497        assert_eq!(
15498            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15499            head
15500        );
15501    }
15502
15503    #[cfg(unix)]
15504    #[tokio::test]
15505    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
15506        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15507        let mut old_review = review_round(true, 0, 1, 1, false, true);
15508        old_review.head = head.clone();
15509        old_review.verified_head = Some(head.clone());
15510        runner.state.reviews = vec![old_review];
15511        runner.state.gate = vec![CommandOutcome {
15512            command: "old gate".into(),
15513            code: None,
15514            output_tail: "timed out after 2400s".into(),
15515            duration_ms: 2_400_000,
15516            resource_blocked: false,
15517        }];
15518        runner.state.gate_ran = true;
15519        runner.state.gate_head = Some(head.clone());
15520        runner.state.land_approval = Some(crate::run::LandApproval {
15521            question: "old-head-approval".into(),
15522            head: head.clone(),
15523        });
15524        runner.state.authorize_base_sync(&task).unwrap();
15525        runner.sync_to_base().await.unwrap();
15526        assert!(!runner.state.base_sync_blocked());
15527        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15528        assert_ne!(head, fresh);
15529        assert!(runner.state.reviews.is_empty());
15530        assert!(!runner.state.gate_ran);
15531        let saved = &runner.state.base_sync_recovery.authorizations[0];
15532        assert_eq!(saved.reviews[0].head, head);
15533        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
15534        assert!(saved.gate_ran);
15535        assert!(!saved.gate[0].ok());
15536        runner.review_loop().await.unwrap();
15537        runner.gate().await.unwrap();
15538        assert_eq!(runner.state.reviews[0].head, fresh);
15539        assert_eq!(
15540            runner.state.reviews[0].verified_head.as_deref(),
15541            Some(fresh.as_str())
15542        );
15543        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
15544        assert!(runner.state.gate_ran);
15545        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15546        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
15547        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
15548        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15549        assert_eq!(
15550            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
15551            "retained change"
15552        );
15553    }
15554
15555    #[cfg(unix)]
15556    #[tokio::test]
15557    async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
15558        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15559        let queue = crate::queue::Queue::open();
15560        let mut owner = queue.get(&task).unwrap();
15561        owner.release_operator("test-resume", None);
15562        queue.put(&mut owner).unwrap();
15563        let generation = owner.resumption.as_ref().unwrap().generation;
15564        runner.state.task_instruction = Some(owner.instruction.clone());
15565        runner.state.driver_exited = true;
15566        runner.state.save().unwrap();
15567        let home = crate::run::home();
15568        let _claim = queue.claim(&task).unwrap();
15569        owner.resumption.as_mut().unwrap().at =
15570            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
15571        queue.put(&mut owner).unwrap();
15572        assert!(
15573            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15574                .await
15575                .is_err(),
15576            "an approval from before the stop grants nothing"
15577        );
15578        owner.resumption.as_mut().unwrap().at = Timestamp::now();
15579        queue.put(&mut owner).unwrap();
15580        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15581            .await
15582            .unwrap();
15583        assert!(
15584            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15585                .await
15586                .is_err()
15587        );
15588        runner.state = RunState::load(&runner.state.id).unwrap();
15589        assert_eq!(
15590            runner.state.base_sync_recovery.authorizations[0].generation,
15591            Some(generation)
15592        );
15593        runner.sync_to_base().await.unwrap();
15594        let repo = runner.state.repo.clone();
15595        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15596        git::git(&repo, &["checkout", "main"]).await.unwrap();
15597        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
15598        git::git(&repo, &["add", "."]).await.unwrap();
15599        git::git(&repo, &["commit", "-m", "move base again"])
15600            .await
15601            .unwrap();
15602        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15603        git::git(&repo, &["checkout", "fix/retained"])
15604            .await
15605            .unwrap();
15606        runner.finish_after_tally().await.unwrap();
15607        assert_eq!(runner.state.status, RunStatus::Blocked);
15608        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15609        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15610        assert!(!runner.state.base_recovery_pending());
15611        assert!(!crate::handover::can_resume(&runner.state));
15612        assert!(!runner.state.gate_ran);
15613        // Even replaying the generation with a later timestamp grants nothing.
15614        owner.resumption.as_mut().unwrap().at = Timestamp::now();
15615        queue.put(&mut owner).unwrap();
15616        assert!(
15617            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15618                .await
15619                .is_err()
15620        );
15621        runner.state = RunState::load(&runner.state.id).unwrap();
15622        runner.sync_to_base().await.unwrap();
15623        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15624        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
15625        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15626    }
15627
15628    #[cfg(unix)]
15629    #[tokio::test]
15630    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
15631        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
15632        runner.state.base_sync = None;
15633        runner.state.status = RunStatus::Blocked;
15634        let mut second = runner.state.candidates[0].clone();
15635        second.label = 'B';
15636        runner.state.candidates.push(second);
15637        assert!(runner.state.votes.is_empty());
15638        assert!(runner.state.deliberation.is_empty());
15639        runner.deliberate().await.unwrap();
15640        runner.vote().await.unwrap();
15641        assert_eq!(runner.state.status, RunStatus::Blocked);
15642        assert!(runner.state.votes.is_empty());
15643        assert!(runner.state.deliberation.is_empty());
15644    }
15645
15646    /// A minimal, git-free `Runner` at a given status — `settle_questions`
15647    /// reads nothing else off it.
15648    fn runner_at(status: RunStatus) -> Runner {
15649        let mut state = RunState::new(
15650            PathBuf::from("/nonexistent/repo"),
15651            "main".to_owned(),
15652            "deadbeef".to_owned(),
15653            "task".to_owned(),
15654            Config::default(),
15655        );
15656        // These fixtures share one home and save concurrently. Short clock
15657        // IDs can collide, making an ownership test read another test's run.
15658        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
15659        state.id = format!(
15660            "20000101-000000-{:04x}",
15661            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
15662        );
15663        state.status = status;
15664        Runner {
15665            state,
15666            roles: ResolvedRoles {
15667                implementers: Vec::new(),
15668                judges: Vec::new(),
15669                reviewers: Vec::new(),
15670                fixer: None,
15671                conductor: conductor(),
15672                implementer_roster: Vec::new(),
15673                judge_roster: Vec::new(),
15674                reviewer_roster: Vec::new(),
15675            },
15676            sem: Arc::new(Semaphore::new(1)),
15677            pause: Pause::new(),
15678            interrupt: Pause::new(),
15679        }
15680    }
15681
15682    #[tokio::test]
15683    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
15684        crate::run::pin_test_home();
15685        let mut runner = runner_at(RunStatus::Gating);
15686        runner.state.save().unwrap();
15687        let id = runner.state.id.clone();
15688        let passes = runner.state.passes.len();
15689        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15690        let mut claiming = Box::pin(runner.claim_driver());
15691        std::future::poll_fn(|cx| {
15692            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
15693            std::task::Poll::Ready(())
15694        })
15695        .await;
15696        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
15697        drop(owner);
15698        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
15699            .await
15700            .unwrap()
15701            .unwrap();
15702        assert_eq!(execution.runner.state.passes.len(), passes + 1);
15703        assert!(execution.runner.state.resource_block.is_none());
15704        assert_eq!(
15705            execution.runner.state.liveness(false),
15706            crate::run::Liveness::Live
15707        );
15708        drop(execution);
15709        assert!(RunState::load(&id).unwrap().driver_exited);
15710    }
15711
15712    #[tokio::test]
15713    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
15714        crate::run::pin_test_home();
15715        let mut runner = runner_at(RunStatus::Gating);
15716        runner.state.save().unwrap();
15717        // run.json can commit, but the subsequent lease metadata write fails.
15718        // Cleanup must run after releasing the short ownership lock.
15719        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
15720        assert!(runner.claim_driver().await.is_err());
15721        assert!(runner.state.driver_exited);
15722        assert_eq!(runner.state.driver_pid, None);
15723        let disk = RunState::load(&runner.state.id).unwrap();
15724        assert!(disk.driver_exited);
15725        assert_eq!(disk.driver_pid, None);
15726        assert_eq!(disk.status, RunStatus::Gating);
15727    }
15728
15729    #[tokio::test]
15730    async fn an_early_graph_error_releases_its_driver() {
15731        crate::run::pin_test_home();
15732        let mut runner = runner_at(RunStatus::Blocked);
15733        runner.state.config.graph.selected_implementer = Some("selected".into());
15734        runner.state.save().unwrap();
15735        assert!(runner.execute().await.is_err());
15736        let disk = RunState::load(&runner.state.id).unwrap();
15737        assert!(disk.driver_exited);
15738        assert_eq!(disk.driver_pid, None);
15739        assert_eq!(disk.status, RunStatus::Blocked);
15740    }
15741
15742    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
15743        crate::run::MergeRepair {
15744            question: "repair-question".into(),
15745            head: "old-head".into(),
15746            pr: "https://github.com/o/r/pull/42".into(),
15747            reviews: vec![review_round(false, 1, 1, 1, false, true)],
15748            gate: vec![outcome(Some(0), false)],
15749            gate_head: Some("old-head".into()),
15750            gate_fixes: Vec::new(),
15751            pre_gate: Vec::new(),
15752            phase,
15753            failure: None,
15754        }
15755    }
15756
15757    #[tokio::test]
15758    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
15759        crate::run::pin_test_home();
15760        let mut runner = runner_at(RunStatus::Landing);
15761        runner.state.merge_repairs.push(merge_repair_request(
15762            crate::run::MergeRepairPhase::Requested,
15763        ));
15764        runner.state.save().unwrap();
15765        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
15766        assert!(
15767            runner
15768                .execute()
15769                .await
15770                .unwrap_err()
15771                .to_string()
15772                .contains("already running")
15773        );
15774        assert_eq!(
15775            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
15776            crate::run::MergeRepairPhase::Requested
15777        );
15778        drop(claim);
15779        let mut disk = runner.state.clone();
15780        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
15781        disk.save().unwrap();
15782        assert!(
15783            runner
15784                .execute()
15785                .await
15786                .unwrap_err()
15787                .to_string()
15788                .contains("decision changed")
15789        );
15790        assert_eq!(
15791            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
15792            crate::run::MergeRepairPhase::Complete
15793        );
15794    }
15795
15796    #[test]
15797    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
15798        crate::run::pin_test_home();
15799        let mut runner = runner_at(RunStatus::Landing);
15800        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
15801        runner.state.reviews = request.reviews.clone();
15802        runner.state.gate = request.gate.clone();
15803        runner.state.gate_ran = true;
15804        runner.state.land_approval = Some(crate::run::LandApproval {
15805            question: "old".into(),
15806            head: "old-head".into(),
15807        });
15808        runner.state.merge_repairs.push(request);
15809        runner.begin_merge_repair_review(0).unwrap();
15810        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
15811        assert!(!runner.state.gate_status().ok());
15812        assert!(runner.state.land_approval.is_none());
15813        let disk = RunState::load(&runner.state.id).unwrap();
15814        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
15815        assert!(disk.merge_repairs[0].gate[0].ok());
15816        assert_eq!(
15817            disk.merge_repairs[0].phase,
15818            crate::run::MergeRepairPhase::Reviewing
15819        );
15820        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
15821        assert!(disk.reviews.is_empty());
15822    }
15823
15824    #[tokio::test]
15825    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
15826        crate::run::pin_test_home();
15827        for phase in [
15828            crate::run::MergeRepairPhase::Fixing,
15829            crate::run::MergeRepairPhase::Requested,
15830        ] {
15831            let mut runner = runner_at(RunStatus::Landing);
15832            runner.state.config.graph.land_rounds = 0;
15833            runner.state.merge_repairs.push(merge_repair_request(phase));
15834            runner.state.save().unwrap();
15835            assert!(!runner.repair_merge_review().await.unwrap());
15836            assert_eq!(runner.state.status, RunStatus::Blocked);
15837            assert_eq!(
15838                runner.state.merge_repairs[0].phase,
15839                crate::run::MergeRepairPhase::Failed
15840            );
15841            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
15842            assert!(
15843                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
15844                    "interrupted"
15845                } else {
15846                    "budget exhausted"
15847                })
15848            );
15849            assert!(runner.state.merge.is_none());
15850        }
15851    }
15852
15853    /// `park_here` folding in the reason `Pause::park_because` recorded -
15854    /// this is what lets an operator reading a run's events tell an
15855    /// interrupt-driven park from an ordinary shutdown park.
15856    #[test]
15857    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15858        crate::run::pin_test_home();
15859        let mut runner = runner_at(RunStatus::Implementing);
15860        let interrupt = Pause::new();
15861        runner.watch_interrupt(interrupt.clone());
15862
15863        interrupt.park_because("task a1b2 asked to run first");
15864
15865        assert!(runner.park_here().expect("park_here"));
15866        assert!(runner.state.parked);
15867        let last = runner.state.events.last().expect("a park event");
15868        assert_eq!(last.node, "park");
15869        assert!(
15870            last.message.contains("task a1b2 asked to run first"),
15871            "expected the interrupt reason in {:?}",
15872            last.message
15873        );
15874    }
15875
15876    /// The cause is stored on the pass when it parks, and the first cause a
15877    /// shared pause was given is the one that stays.
15878    #[test]
15879    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15880        crate::run::pin_test_home();
15881        let mut runner = runner_at(RunStatus::Implementing);
15882        runner.state.begin_pass();
15883        let shutdown = Pause::new();
15884        runner.on_pause(shutdown.clone());
15885        let upgrade = ParkCause::Upgrade {
15886            from: "0.121.0".to_owned(),
15887            to: Some("0.121.1".to_owned()),
15888        };
15889        shutdown.park_for(upgrade.clone());
15890        shutdown.park_for(ParkCause::Operator);
15891
15892        assert!(runner.park_here().expect("park_here"));
15893        assert_eq!(
15894            runner.state.passes[0].end,
15895            Some(crate::run::PassEnd::Parked { cause: upgrade })
15896        );
15897        let last = runner.state.events.last().expect("a park event");
15898        assert!(last.message.contains("0.121.0"), "{}", last.message);
15899    }
15900
15901    #[tokio::test]
15902    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15903        crate::run::pin_test_home();
15904        let mut runner = runner_at(RunStatus::Reviewing);
15905        runner.state.save().unwrap();
15906        let mut execution = runner.claim_driver().await.expect("first execution");
15907        assert_eq!(execution.runner.state.passes.len(), 1);
15908        execution
15909            .driver
15910            .finish(&mut execution.runner.state)
15911            .await
15912            .unwrap();
15913    }
15914
15915    #[tokio::test]
15916    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15917        crate::run::pin_test_home();
15918        let mut runner = runner_at(RunStatus::Reviewing);
15919        runner.state.config.agents = vec![conductor()];
15920        runner.state.config.graph.timeout_resource_wait = 0;
15921        runner.state.save().unwrap();
15922        let id = runner.state.id.clone();
15923        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15924        assert!(runner.claim_driver().await.is_err());
15925        drop(owner);
15926        drop(runner);
15927        let mut resumed = Runner::resume(&id).unwrap();
15928        assert_eq!(resumed.state.driver_claimed, Some(false));
15929        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15930        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15931        let mut execution = resumed
15932            .claim_driver()
15933            .await
15934            .expect("retry initial claim after reload");
15935        assert_eq!(execution.runner.state.driver_claimed, Some(true));
15936        assert_eq!(execution.runner.state.passes.len(), 1);
15937        execution
15938            .driver
15939            .finish(&mut execution.runner.state)
15940            .await
15941            .unwrap();
15942    }
15943
15944    #[tokio::test]
15945    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15946        crate::run::pin_test_home();
15947        let mut runner = runner_at(RunStatus::Reviewing);
15948        runner.state.config.agents = vec![conductor()];
15949        runner.state.save().unwrap();
15950        let id = runner.state.id.clone();
15951        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15952        let mut resumed = Runner::resume(&id).unwrap();
15953        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15954        assert!(resumed.claim_driver().await.is_err());
15955        assert!(RunState::load(&id).unwrap().passes.is_empty());
15956        drop(lease);
15957        let mut execution = resumed.claim_driver().await.unwrap();
15958        execution
15959            .driver
15960            .finish(&mut execution.runner.state)
15961            .await
15962            .unwrap();
15963    }
15964
15965    #[tokio::test]
15966    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15967        crate::run::pin_test_home();
15968        let mut runner = runner_at(RunStatus::Reviewing);
15969        let state = &mut runner.state;
15970        state.config.agents = vec![conductor()];
15971        state.driver_claimed = None;
15972        state.driver_pid = Some(std::process::id());
15973        state.driver_started_at = None;
15974        state.begin_pass();
15975        state.save().unwrap();
15976        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15977        std::fs::write(
15978            state.dir().join("driver.json"),
15979            serde_json::to_vec(&serde_json::json!({
15980                "pid": state.driver_pid, "started_at": null,
15981                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15982            }))
15983            .unwrap(),
15984        )
15985        .unwrap();
15986        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15987        drop(lease);
15988        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15989        let mut resumed = Runner::resume(&state.id).unwrap();
15990        assert!(!resumed.state.driver_exited);
15991        let mut execution = resumed
15992            .claim_driver()
15993            .await
15994            .expect("lease proves previous execution ended");
15995        assert_eq!(execution.runner.state.passes.len(), 2);
15996        execution
15997            .driver
15998            .finish(&mut execution.runner.state)
15999            .await
16000            .unwrap();
16001    }
16002
16003    #[tokio::test]
16004    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16005        crate::run::pin_test_home();
16006        let mut runner = runner_at(RunStatus::Reviewing);
16007        let mut json = serde_json::to_value(&runner.state).unwrap();
16008        json.as_object_mut().unwrap().remove("driver_claimed");
16009        runner.state = serde_json::from_value(json).unwrap();
16010        assert_eq!(runner.state.driver_claimed, None);
16011        runner.state.save().unwrap();
16012        let mut execution = runner
16013            .claim_driver()
16014            .await
16015            .expect("the execution lease, not a missing marker, excludes a second driver");
16016        assert_eq!(execution.runner.state.passes.len(), 1);
16017        execution
16018            .driver
16019            .finish(&mut execution.runner.state)
16020            .await
16021            .unwrap();
16022    }
16023
16024    #[tokio::test]
16025    async fn driver_admission_refuses_a_proven_live_driver() {
16026        crate::run::pin_test_home();
16027        let mut runner = runner_at(RunStatus::Reviewing);
16028        let pid = std::process::id();
16029        runner.state.driver_pid = Some(pid);
16030        runner.state.driver_started_at = crate::proc::process_started_at(pid);
16031        assert!(runner.state.driver_started_at.is_some());
16032        runner.state.save().unwrap();
16033        let error = runner
16034            .claim_driver()
16035            .await
16036            .err()
16037            .expect("a live driver must refuse");
16038        assert!(error.to_string().contains("live driver"), "{error:#}");
16039        let saved = RunState::load(&runner.state.id).unwrap();
16040        assert_eq!(saved.driver_pid, Some(pid));
16041        assert_eq!(saved.status, RunStatus::Reviewing);
16042        assert!(saved.passes.is_empty());
16043    }
16044
16045    #[test]
16046    fn parking_resource_halt_persists_the_pass_once() {
16047        crate::run::pin_test_home();
16048        let mut runner = runner_at(RunStatus::Reviewing);
16049        runner.state.begin_pass();
16050        runner.state.resource_block = Some(crate::run::ResourceBlock {
16051            node: "fix".into(),
16052            kind: "shared build cache".into(),
16053            detail: "cancelled".into(),
16054            at: Timestamp::now(),
16055            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16056        });
16057        let cause = ParkCause::Upgrade {
16058            from: "old".into(),
16059            to: Some("new".into()),
16060        };
16061        runner.pause.park_for(cause.clone());
16062        // Both direct fixer exits and graph resource boundaries must complete
16063        // parking, rather than returning a stale Reviewing record.
16064        runner.stop_for_resource().unwrap();
16065        assert!(runner.resource_halt().unwrap());
16066        assert!(runner.park_here().unwrap());
16067        let saved = RunState::load(&runner.state.id).unwrap();
16068        assert!(saved.parked);
16069        assert_eq!(saved.status, RunStatus::Reviewing);
16070        assert!(saved.passes[0].ended_at.is_some());
16071        assert_eq!(
16072            saved.passes[0].end,
16073            Some(crate::run::PassEnd::Parked {
16074                cause: cause.clone()
16075            })
16076        );
16077        assert_eq!(
16078            saved.resource_block.unwrap().cause,
16079            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16080        );
16081        assert_eq!(
16082            saved
16083                .events
16084                .iter()
16085                .filter(|event| event.node == "park")
16086                .count(),
16087            1
16088        );
16089    }
16090
16091    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
16092    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
16093    /// cleared) must not make a *different* run - one only watching its own,
16094    /// unshared interrupt `Pause` - see itself as parked. If a future change
16095    /// ever collapsed these back into one handle, the interrupt scheduler
16096    /// would park every run for the rest of the daemon's life, not just the
16097    /// one it meant to interrupt.
16098    #[test]
16099    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16100        crate::run::pin_test_home();
16101        let mut runner = runner_at(RunStatus::Implementing);
16102        let shutdown = Pause::new();
16103        runner.on_pause(shutdown.clone());
16104        let interrupt = Pause::new();
16105        runner.watch_interrupt(interrupt.clone());
16106
16107        // Nobody has asked for anything yet.
16108        assert!(!runner.park_here().expect("park_here"));
16109        assert!(!runner.state.parked);
16110
16111        // Only the interrupt handle fires; the shutdown handle stays clear.
16112        interrupt.park_because("test");
16113        assert!(!shutdown.parked());
16114        assert!(runner.park_here().expect("park_here"));
16115    }
16116
16117    /// The property every prior attempt at this feature failed to pin down:
16118    /// asking a run to park while one of its nodes has a real, in-flight
16119    /// async operation running (an agent call, in production) must not cut
16120    /// that operation short. `park_here` is only ever consulted *between*
16121    /// `execute`'s node calls - see its own doc - so nothing inside a node
16122    /// can observe a park request until the node itself returns. This proves
16123    /// that structurally, with real `tokio` concurrency and a channel
16124    /// handshake (never a sleep, which would only prove "usually", not
16125    /// "cannot"): the "node" below reports that it has genuinely started,
16126    /// and only then is the park requested; the node still has to be told to
16127    /// finish before `park_here` is ever called, exactly mirroring every
16128    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
16129    /// in `execute`.
16130    #[tokio::test]
16131    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16132        crate::run::pin_test_home();
16133        let mut runner = runner_at(RunStatus::Implementing);
16134        let interrupt = Pause::new();
16135        runner.watch_interrupt(interrupt.clone());
16136
16137        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16138        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16139
16140        // Stands in for one node's in-flight agent call: it proves it has
16141        // genuinely started, then blocks - exactly as a spawned CLI process
16142        // does - until told to finish.
16143        let node = async move {
16144            started_tx.send(()).expect("send started");
16145            finish_rx.await.expect("recv finish");
16146            "node finished"
16147        };
16148
16149        let interrupter = async move {
16150            started_rx.await.expect("recv started");
16151            // The call is now genuinely in flight. Ask it to park.
16152            interrupt.park_because("higher-priority task waiting");
16153            // Nothing the node does can observe this yet - there is no
16154            // check inside it, by construction - so let the executor run
16155            // anything pending and then let the node finish on its own.
16156            tokio::task::yield_now().await;
16157            finish_tx.send(()).expect("send finish");
16158        };
16159
16160        let (node_result, ()) = tokio::join!(node, interrupter);
16161        assert_eq!(
16162            node_result, "node finished",
16163            "the in-flight call ran to completion"
16164        );
16165
16166        // Only now, at the boundary the real `execute` would check right
16167        // after this node, does the park take effect.
16168        assert!(runner.park_here().expect("park_here"));
16169        assert!(runner.state.parked);
16170    }
16171
16172    /// A run parked mid-competition carries every field it had accumulated
16173    /// through the exact same disk round-trip an ordinary resume uses -
16174    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
16175    /// Nothing about parking for an interrupt is a special case of that path;
16176    /// this is what proves it rather than assuming it.
16177    #[test]
16178    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16179        crate::run::pin_test_home();
16180        let mut runner = runner_at(RunStatus::Judging);
16181        // `Runner::resume` re-resolves roles from the saved config, which
16182        // refuses an empty roster - give it the same minimal one `conductor`
16183        // itself uses.
16184        runner.state.config.agents = vec![conductor()];
16185        runner.state.candidates = vec![Candidate {
16186            index: 0,
16187            label: 'A',
16188            agent: "alpha".to_owned(),
16189            branch: "magi/x/A".to_owned(),
16190            worktree: PathBuf::from("/nonexistent/worktree"),
16191            summary: "did the thing".to_owned(),
16192            stat: "1 file changed".to_owned(),
16193            files: 1,
16194            commits: 1,
16195            empty: false,
16196            failed: None,
16197            verified_noop: None,
16198            duration_ms: 1234,
16199            folded: false,
16200        }];
16201        let run_id = runner.state.id.clone();
16202
16203        let interrupt = Pause::new();
16204        runner.watch_interrupt(interrupt.clone());
16205        interrupt.park_because("task c3d4 asked to run first");
16206        assert!(runner.park_here().expect("park_here"));
16207
16208        let resumed = Runner::resume(&run_id).expect("resume");
16209        assert_eq!(resumed.state.candidates.len(), 1);
16210        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16211        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16212        assert_eq!(resumed.state.status, runner.state.status);
16213        assert!(
16214            resumed.state.parked,
16215            "still parked until `execute` actually walks the graph again"
16216        );
16217        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16218    }
16219
16220    /// A fresh open question on `run`, stored and handed back for assertions.
16221    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16222        let mut q = ask::Question::new(
16223            run.to_owned(),
16224            "implement".to_owned(),
16225            "impl-A".to_owned(),
16226            "Which storage backend should the cache use?".to_owned(),
16227            String::new(),
16228            vec!["SQLite".to_owned(), "Redis".to_owned()],
16229        );
16230        store.put(&mut q).unwrap();
16231        q
16232    }
16233
16234    #[test]
16235    fn a_failed_runs_open_question_is_abandoned() {
16236        ask_test_home();
16237        let store = ask::Questions::open();
16238        let mut runner = runner_at(RunStatus::Failed);
16239        let run = runner.state.id.clone();
16240        let q = ask_open_question(&store, &run);
16241
16242        runner.settle_questions();
16243
16244        let back = store.get(&q.id).unwrap();
16245        assert!(
16246            !back.status.open(),
16247            "the seat that asked died with the run; nobody is left to read an answer"
16248        );
16249        assert!(
16250            back.detail.contains(&run) && back.detail.contains("failed"),
16251            "the reason names what the run became, not just that it is gone: {}",
16252            back.detail
16253        );
16254    }
16255
16256    #[test]
16257    fn a_merged_runs_open_question_is_abandoned_too() {
16258        ask_test_home();
16259        let store = ask::Questions::open();
16260        // A run that finishes cleanly still leaves nobody to read an answer -
16261        // this is not only a failure-path cleanup.
16262        for status in [RunStatus::Merged, RunStatus::Ready] {
16263            let mut runner = runner_at(status);
16264            let run = runner.state.id.clone();
16265            let q = ask_open_question(&store, &run);
16266
16267            runner.settle_questions();
16268
16269            let back = store.get(&q.id).unwrap();
16270            assert!(
16271                !back.status.open(),
16272                "{status:?} run's question must not outlive the run"
16273            );
16274        }
16275    }
16276
16277    #[test]
16278    fn a_still_resumable_runs_open_question_is_left_alone() {
16279        ask_test_home();
16280        let store = ask::Questions::open();
16281        // `Blocked` and `Stalled` can still be resumed — the candidates, the
16282        // review round and the seat sessions are all still on disk — so a
16283        // question asked mid-round may yet get a real answer from a real
16284        // resume. Sweeping it here would be exactly the failure mode this
16285        // whole feature exists to avoid on the other side.
16286        for status in [RunStatus::Blocked, RunStatus::Stalled] {
16287            let mut runner = runner_at(status);
16288            let run = runner.state.id.clone();
16289            let q = ask_open_question(&store, &run);
16290
16291            runner.settle_questions();
16292
16293            let back = store.get(&q.id).unwrap();
16294            assert!(
16295                back.status.open(),
16296                "{status:?} is still alive; the question must still be waiting"
16297            );
16298        }
16299    }
16300
16301    #[test]
16302    fn settle_questions_never_touches_an_already_answered_question() {
16303        ask_test_home();
16304        let store = ask::Questions::open();
16305        let mut runner = runner_at(RunStatus::Failed);
16306        let run = runner.state.id.clone();
16307        let mut q = ask_open_question(&store, &run);
16308        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16309            .unwrap();
16310        store.put(&mut q).unwrap();
16311
16312        // Called twice, the way a crash-recovered daemon reclaim and the
16313        // graph's own cleanup both can for the same run — `abandon_for_run`
16314        // only ever touches what is still open, so this must be inert both
16315        // times, not merely the second.
16316        runner.settle_questions();
16317        runner.settle_questions();
16318
16319        let back = store.get(&q.id).unwrap();
16320        assert_eq!(
16321            back.status,
16322            ask::QuestionStatus::Answered,
16323            "a real answer is a decision on record, never overwritten by a sweep"
16324        );
16325    }
16326
16327    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
16328    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
16329    /// without merging, whose winner is still the operator's answer to read.
16330    /// Nothing previously called `fold_run` itself with a real `tally`, so
16331    /// this is the first test to pin down the one distinction the whole
16332    /// automatic-fold feature depends on: the winner's worktree and branch
16333    /// must survive, everything else sharing the run's worktree bay - a
16334    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
16335    /// second sweep treats every non-winner directory under the bay alike -
16336    /// must not.
16337    #[tokio::test]
16338    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16339        crate::run::pin_test_home();
16340        let tmp = tempfile::tempdir().expect("tempdir");
16341        let repo = tmp.path().join("repo");
16342        std::fs::create_dir_all(&repo).unwrap();
16343        init_repo(&repo);
16344
16345        let mut config = Config::default();
16346        config.graph.worktree_root = Some(tmp.path().join("wt"));
16347
16348        let mut state = RunState::new(
16349            repo.clone(),
16350            "main".to_owned(),
16351            "deadbeef".to_owned(),
16352            "task".to_owned(),
16353            config,
16354        );
16355        let root = state.worktree_root();
16356        let wt_a = root.join("cand-A");
16357        let wt_b = root.join("cand-B");
16358        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16359            .await
16360            .expect("worktree A");
16361        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16362            .await
16363            .expect("worktree B");
16364
16365        state.candidates = vec![
16366            Candidate {
16367                index: 0,
16368                label: 'A',
16369                agent: "alpha".to_owned(),
16370                branch: "magi/x/A".to_owned(),
16371                worktree: wt_a.clone(),
16372                summary: String::new(),
16373                stat: String::new(),
16374                files: 0,
16375                commits: 0,
16376                empty: false,
16377                failed: None,
16378                verified_noop: None,
16379                duration_ms: 0,
16380                folded: false,
16381            },
16382            Candidate {
16383                index: 1,
16384                label: 'B',
16385                agent: "beta".to_owned(),
16386                branch: "magi/x/B".to_owned(),
16387                worktree: wt_b.clone(),
16388                summary: String::new(),
16389                stat: String::new(),
16390                files: 0,
16391                commits: 0,
16392                empty: false,
16393                failed: None,
16394                verified_noop: None,
16395                duration_ms: 0,
16396                folded: false,
16397            },
16398        ];
16399        state.tally = Some(Tally {
16400            first_choice: BTreeMap::from([('A', 1)]),
16401            borda: BTreeMap::new(),
16402            winner: 'A',
16403            rankings: 1,
16404            unanimous_initial: true,
16405            deliberated: false,
16406            changed_votes: 0,
16407            unanimous_final: true,
16408            tie_break: None,
16409            judges: 1,
16410            present: 1,
16411            quorum: 1,
16412            met_quorum: true,
16413            uncontested: None,
16414        });
16415        state.status = RunStatus::Ready;
16416
16417        fold_run(&mut state, false, &crate::run::home())
16418            .await
16419            .expect("fold_run");
16420
16421        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
16422        assert!(
16423            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16424            "the unmerged winner's branch survives"
16425        );
16426        assert!(
16427            !state.candidates[0].folded,
16428            "the winner is not marked folded"
16429        );
16430
16431        assert!(!wt_b.exists(), "the loser's worktree is removed");
16432        assert!(
16433            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
16434            "the loser's branch is removed"
16435        );
16436        assert!(state.candidates[1].folded, "the loser is marked folded");
16437    }
16438
16439    #[tokio::test]
16440    #[cfg(unix)]
16441    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
16442        use std::os::fd::AsRawFd as _;
16443        use std::time::Duration;
16444
16445        let tmp = tempfile::tempdir().unwrap();
16446        let repo = tmp.path().join("repo");
16447        std::fs::create_dir_all(&repo).unwrap();
16448        init_repo(&repo);
16449        let common = repo.join(".git");
16450        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
16451        let home = tmp.path().join("home");
16452        let mut config = Config::default();
16453        config.coordination = crate::config::test_coordination();
16454        config.graph.worktree_root = Some(tmp.path().join("wt"));
16455        config.graph.timeout_resource_wait = 1;
16456        let mut state = RunState::new(
16457            repo.clone(),
16458            "main".into(),
16459            "deadbeef".into(),
16460            "task".into(),
16461            config,
16462        );
16463        let root = state.worktree_root();
16464        let candidate = root.join("cand-A");
16465        let reviewer = root.join("review-1");
16466        let branch = state.branch_for('A');
16467        git::worktree_add_branch(&repo, &candidate, &branch, "main")
16468            .await
16469            .unwrap();
16470        git::worktree_add_detached(&repo, &reviewer, "main")
16471            .await
16472            .unwrap();
16473        git::acquire_worktree_config(&repo).await.unwrap();
16474        state.enabled_worktree_config = true;
16475        state.status = RunStatus::Ready;
16476        state.candidates.push(Candidate {
16477            index: 0,
16478            label: 'A',
16479            agent: "implement".into(),
16480            branch: branch.clone(),
16481            worktree: candidate.clone(),
16482            summary: String::new(),
16483            stat: String::new(),
16484            files: 0,
16485            commits: 0,
16486            empty: false,
16487            failed: None,
16488            verified_noop: None,
16489            duration_ms: 0,
16490            folded: false,
16491        });
16492        state.save_under(&home).unwrap();
16493        let config_before = std::fs::read(common.join("config")).unwrap();
16494        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
16495            .await
16496            .unwrap();
16497        // A separate OS handle bypasses the launcher's local gate, exactly as
16498        // an external updater does, and must still prevent all folding work.
16499        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
16500        assert_eq!(
16501            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
16502            0
16503        );
16504        let mut fold = Box::pin(fold_run(&mut state, true, &home));
16505        assert!(
16506            tokio::time::timeout(Duration::from_millis(200), &mut fold)
16507                .await
16508                .is_err()
16509        );
16510        assert!(candidate.exists());
16511        assert!(reviewer.exists());
16512        assert!(git::branch_exists(&repo, &branch).await.unwrap());
16513        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
16514        assert_eq!(
16515            git::git(&repo, &["worktree", "list", "--porcelain"])
16516                .await
16517                .unwrap(),
16518            worktrees_before
16519        );
16520        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
16521            .await
16522            .unwrap()
16523            .unwrap_err();
16524        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
16525        drop(fold);
16526        assert!(candidate.exists() && reviewer.exists());
16527        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
16528        assert_eq!(
16529            state.resource_block.as_ref().unwrap().kind,
16530            "repository coordination"
16531        );
16532        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
16533        drop(external);
16534        let mut fold = Box::pin(fold_run(&mut state, true, &home));
16535        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
16536            .await
16537            .unwrap()
16538            .unwrap();
16539        drop(fold);
16540        assert!(removed.contains(&branch));
16541        assert!(!root.exists());
16542        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
16543        assert!(state.candidates[0].folded);
16544        assert!(!state.enabled_worktree_config);
16545        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
16546        let enabled = git::git_raw(
16547            &repo,
16548            &["config", "--local", "--get", "extensions.worktreeConfig"],
16549        )
16550        .await
16551        .unwrap();
16552        assert!(!enabled.ok(), "the last config reference was released");
16553    }
16554
16555    /// A branch handed to a later run is that run's (and its pull request's):
16556    /// folding the run that released it must not delete it.
16557    #[tokio::test]
16558    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
16559        let tmp = tempfile::tempdir().expect("tempdir");
16560        let repo = tmp.path().join("repo");
16561        std::fs::create_dir_all(&repo).unwrap();
16562        init_repo(&repo);
16563        let home = tmp.path().join("home");
16564
16565        let mut config = Config::default();
16566        config.graph.worktree_root = Some(tmp.path().join("wt"));
16567        let mut state = RunState::new(
16568            repo.clone(),
16569            "main".to_owned(),
16570            "deadbeef".to_owned(),
16571            "task".to_owned(),
16572            config,
16573        );
16574        // The worktree is already gone (released); the branch survives.
16575        git::git(&repo, &["branch", "magi/x/A", "main"])
16576            .await
16577            .expect("branch");
16578        state.candidates = vec![Candidate {
16579            index: 0,
16580            label: 'A',
16581            agent: "alpha".to_owned(),
16582            branch: "magi/x/A".to_owned(),
16583            worktree: state.worktree_root().join("cand-A"),
16584            summary: String::new(),
16585            stat: String::new(),
16586            files: 0,
16587            commits: 0,
16588            empty: false,
16589            failed: None,
16590            verified_noop: None,
16591            duration_ms: 0,
16592            folded: true,
16593        }];
16594        state.released_to = Some("20260901-000000-new1".to_owned());
16595        state.released_branches = vec!["magi/x/A".to_owned()];
16596
16597        fold_run(&mut state, true, &home).await.expect("fold_run");
16598        state.status = RunStatus::Merged;
16599        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
16600        state.merge = Some(crate::run::MergeOutcome {
16601            mode: MergeMode::Pr,
16602            ok: true,
16603            detail: "confirmed merged".into(),
16604            empty: false,
16605        });
16606        fold_run(&mut state, true, &home)
16607            .await
16608            .expect("merged fold");
16609
16610        assert!(
16611            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16612            "the handed-over branch survives a fold"
16613        );
16614    }
16615
16616    /// A winner with nothing ahead of the base is caught before `gh` is ever
16617    /// asked for a pull request, and the message carries what the task's
16618    /// references resolved to.
16619    #[tokio::test]
16620    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
16621        let tmp = tempfile::tempdir().expect("tempdir");
16622        let repo = tmp.path().join("repo");
16623        std::fs::create_dir_all(&repo).unwrap();
16624        init_repo(&repo);
16625        let run = |args: &[&str]| {
16626            let out = std::process::Command::new("git")
16627                .quiet()
16628                .args(args)
16629                .current_dir(&repo)
16630                .output()
16631                .expect("spawn git");
16632            assert!(out.status.success(), "git {args:?}");
16633        };
16634        run(&["branch", "magi/x/A"]);
16635        run(&["checkout", "-q", "-b", "magi/x/B"]);
16636        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
16637        run(&["add", "-A"]);
16638        run(&["commit", "-q", "-m", "work"]);
16639        run(&["checkout", "-q", "main"]);
16640
16641        // A pull request is compared against the remote's base, so the
16642        // fixture needs one. Before it exists nothing can be read, and the
16643        // answer must be "not empty".
16644        let probe = RunState::new(
16645            repo.clone(),
16646            "main".to_owned(),
16647            "deadbeef".to_owned(),
16648            "task".to_owned(),
16649            Config::default(),
16650        );
16651        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
16652        let bare = tmp.path().join("origin.git");
16653        let out = std::process::Command::new("git")
16654            .quiet()
16655            .args(["init", "-q", "--bare"])
16656            .arg(&bare)
16657            .output()
16658            .expect("spawn git");
16659        assert!(out.status.success(), "git init --bare");
16660        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
16661        run(&["push", "-q", "origin", "main"]);
16662
16663        let mut state = RunState::new(
16664            repo.clone(),
16665            "main".to_owned(),
16666            "deadbeef".to_owned(),
16667            "task".to_owned(),
16668            Config::default(),
16669        );
16670        state.seeds = vec![refs::Seed {
16671            token: "magi/27b2/A".to_owned(),
16672            kind: refs::SeedKind::Unresolved,
16673            sha: String::new(),
16674            branch: true,
16675            detail: "no branch or commit named magi/27b2/A".to_owned(),
16676        }];
16677
16678        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
16679        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
16680        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
16681        let detail = empty_candidate_detail(&state, "main");
16682        assert!(detail.starts_with("empty candidate"), "{detail}");
16683        assert!(!detail.contains("magi/27b2/A"), "{detail}");
16684        assert!(detail.contains("--start-from"), "{detail}");
16685    }
16686
16687    /// `status == Ready` used to be read as "this is the harmless
16688    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
16689    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
16690    /// run's PR was closed without merging — and reentering `merge` with
16691    /// `mode` still `Pr` does not know the difference, so it pushed and
16692    /// opened a second pull request. `mode == Local` reproduces the same
16693    /// blind spot without a network call: reentry must not attempt another
16694    /// git merge once this node has already recorded an outcome.
16695    #[tokio::test]
16696    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
16697        let tmp = tempfile::tempdir().expect("tempdir");
16698        let repo = tmp.path().join("repo");
16699        std::fs::create_dir_all(&repo).unwrap();
16700        init_repo(&repo);
16701
16702        let mut config = Config::default();
16703        config.merge.mode = MergeMode::Local;
16704
16705        let mut state = RunState::new(
16706            repo.clone(),
16707            "main".to_owned(),
16708            "deadbeef".to_owned(),
16709            "task".to_owned(),
16710            config,
16711        );
16712        state.candidates = vec![Candidate {
16713            index: 0,
16714            label: 'A',
16715            agent: "alpha".to_owned(),
16716            branch: "does-not-exist".to_owned(),
16717            worktree: repo.clone(),
16718            summary: String::new(),
16719            stat: String::new(),
16720            files: 0,
16721            commits: 0,
16722            empty: false,
16723            failed: None,
16724            verified_noop: None,
16725            duration_ms: 0,
16726            folded: false,
16727        }];
16728        state.tally = Some(Tally {
16729            first_choice: BTreeMap::from([('A', 1)]),
16730            borda: BTreeMap::new(),
16731            winner: 'A',
16732            rankings: 1,
16733            unanimous_initial: true,
16734            deliberated: false,
16735            changed_votes: 0,
16736            unanimous_final: true,
16737            tie_break: None,
16738            judges: 0,
16739            present: 0,
16740            quorum: 0,
16741            met_quorum: true,
16742            uncontested: Some("only candidate A produced a change".to_owned()),
16743        });
16744        state.reviews = vec![ReviewRound {
16745            round: 1,
16746            head: "deadbeef".to_owned(),
16747            verified_head: None,
16748            verified_at: None,
16749            reviews: Vec::new(),
16750            e2e: Vec::new(),
16751            fix: None,
16752            blocking: 0,
16753            answered: 0,
16754            expected: 0,
16755            clean: true,
16756            verify_retried: false,
16757            e2e_deferred: false,
16758            e2e_defer_reason: None,
16759            progressed: false,
16760            vote_split: false,
16761            reconsideration: Vec::new(),
16762            verdict: None,
16763        }];
16764        state.gate = vec![CommandOutcome {
16765            command: "test".to_owned(),
16766            code: Some(0),
16767            output_tail: String::new(),
16768            duration_ms: 0,
16769            resource_blocked: false,
16770        }];
16771        state.gate_ran = true;
16772        // Reached its conclusion already — e.g. `land` closing the PR without
16773        // merging it, which (like the honest `MergeMode::None` path) leaves
16774        // `status` at `Ready`. The recorded outcome is what actually marks
16775        // this node done.
16776        state.status = RunStatus::Ready;
16777        state.merge = Some(MergeOutcome {
16778            mode: MergeMode::Local,
16779            ok: false,
16780            detail: "already concluded".to_owned(),
16781            empty: false,
16782        });
16783
16784        let mut runner = Runner {
16785            state,
16786            roles: ResolvedRoles {
16787                implementers: Vec::new(),
16788                judges: Vec::new(),
16789                reviewers: Vec::new(),
16790                fixer: None,
16791                conductor: conductor(),
16792                implementer_roster: Vec::new(),
16793                judge_roster: Vec::new(),
16794                reviewer_roster: Vec::new(),
16795            },
16796            sem: Arc::new(Semaphore::new(1)),
16797            pause: Pause::new(),
16798            interrupt: Pause::new(),
16799        };
16800
16801        runner.merge().await.expect("merge");
16802
16803        assert_eq!(
16804            runner.state.status,
16805            RunStatus::Ready,
16806            "a concluded run's status must not change on reentry"
16807        );
16808        assert_eq!(
16809            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16810            Some("already concluded"),
16811            "merge must not run again once the node already recorded an outcome"
16812        );
16813    }
16814
16815    /// `gate` leaves `state.gate_ran` false both before it has ever run and
16816    /// when its last attempt was resource-blocked (the shared build cache
16817    /// could not be acquired or confirmed fresh in time - see
16818    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
16819    /// `Vec` this also leaves behind used to read as "nothing failed" and let
16820    /// a run merge a tree the gate never actually checked - exactly the case
16821    /// a contended cache produces on every retry until it clears. `merge`
16822    /// must refuse until `gate` has actually recorded an attempt.
16823    #[tokio::test]
16824    async fn merge_refuses_a_gate_that_has_not_actually_run() {
16825        let tmp = tempfile::tempdir().expect("tempdir");
16826        let repo = tmp.path().join("repo");
16827        std::fs::create_dir_all(&repo).unwrap();
16828        init_repo(&repo);
16829
16830        let mut config = Config::default();
16831        config.merge.mode = MergeMode::Local;
16832
16833        let mut state = RunState::new(
16834            repo.clone(),
16835            "main".to_owned(),
16836            "deadbeef".to_owned(),
16837            "task".to_owned(),
16838            config,
16839        );
16840        state.candidates = vec![Candidate {
16841            index: 0,
16842            label: 'A',
16843            agent: "alpha".to_owned(),
16844            branch: "does-not-exist".to_owned(),
16845            worktree: repo.clone(),
16846            summary: String::new(),
16847            stat: String::new(),
16848            files: 0,
16849            commits: 0,
16850            empty: false,
16851            failed: None,
16852            verified_noop: None,
16853            duration_ms: 0,
16854            folded: false,
16855        }];
16856        state.tally = Some(Tally {
16857            first_choice: BTreeMap::from([('A', 1)]),
16858            borda: BTreeMap::new(),
16859            winner: 'A',
16860            rankings: 1,
16861            unanimous_initial: true,
16862            deliberated: false,
16863            changed_votes: 0,
16864            unanimous_final: true,
16865            tie_break: None,
16866            judges: 0,
16867            present: 0,
16868            quorum: 0,
16869            met_quorum: true,
16870            uncontested: Some("only candidate A produced a change".to_owned()),
16871        });
16872        state.reviews = vec![ReviewRound {
16873            round: 1,
16874            head: "deadbeef".to_owned(),
16875            verified_head: None,
16876            verified_at: None,
16877            reviews: Vec::new(),
16878            e2e: Vec::new(),
16879            fix: None,
16880            blocking: 0,
16881            answered: 0,
16882            expected: 0,
16883            clean: true,
16884            verify_retried: false,
16885            e2e_deferred: false,
16886            e2e_defer_reason: None,
16887            progressed: false,
16888            vote_split: false,
16889            reconsideration: Vec::new(),
16890            verdict: None,
16891        }];
16892        // The point: `gate` has not recorded anything yet.
16893        state.gate = Vec::new();
16894        state.gate_ran = false;
16895        state.status = RunStatus::Gating;
16896
16897        let mut runner = Runner {
16898            state,
16899            roles: ResolvedRoles {
16900                implementers: Vec::new(),
16901                judges: Vec::new(),
16902                reviewers: Vec::new(),
16903                fixer: None,
16904                conductor: conductor(),
16905                implementer_roster: Vec::new(),
16906                judge_roster: Vec::new(),
16907                reviewer_roster: Vec::new(),
16908            },
16909            sem: Arc::new(Semaphore::new(1)),
16910            pause: Pause::new(),
16911            interrupt: Pause::new(),
16912        };
16913
16914        runner.merge().await.expect("merge");
16915
16916        assert!(
16917            runner.state.merge.is_none(),
16918            "an empty gate must never be read as a passing one: {:?}",
16919            runner.state.merge
16920        );
16921    }
16922
16923    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
16924    /// commands configured and `merge.mode` is `none` (a review-only run).
16925    /// `gate` must still record a real attempt — zero commands, vacuously
16926    /// passed — rather than leaving `state.gate` empty in a way `merge`
16927    /// cannot tell apart from "never ran"; otherwise the run reaches
16928    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
16929    #[tokio::test]
16930    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16931        let tmp = tempfile::tempdir().expect("tempdir");
16932        let repo = tmp.path().join("repo");
16933        std::fs::create_dir_all(&repo).unwrap();
16934        init_repo(&repo);
16935
16936        // Default config: `verify.gate` empty, `merge.mode` is `none`.
16937        let config = Config::default();
16938
16939        let mut state = RunState::new(
16940            repo.clone(),
16941            "main".to_owned(),
16942            "deadbeef".to_owned(),
16943            "task".to_owned(),
16944            config,
16945        );
16946        state.candidates = vec![Candidate {
16947            index: 0,
16948            label: 'A',
16949            agent: "alpha".to_owned(),
16950            branch: "does-not-exist".to_owned(),
16951            worktree: repo.clone(),
16952            summary: String::new(),
16953            stat: String::new(),
16954            files: 0,
16955            commits: 0,
16956            empty: false,
16957            failed: None,
16958            verified_noop: None,
16959            duration_ms: 0,
16960            folded: false,
16961        }];
16962        state.tally = Some(Tally {
16963            first_choice: BTreeMap::from([('A', 1)]),
16964            borda: BTreeMap::new(),
16965            winner: 'A',
16966            rankings: 1,
16967            unanimous_initial: true,
16968            deliberated: false,
16969            changed_votes: 0,
16970            unanimous_final: true,
16971            tie_break: None,
16972            judges: 0,
16973            present: 0,
16974            quorum: 0,
16975            met_quorum: true,
16976            uncontested: Some("only candidate A produced a change".to_owned()),
16977        });
16978        state.reviews = vec![ReviewRound {
16979            round: 1,
16980            head: "deadbeef".to_owned(),
16981            verified_head: None,
16982            verified_at: None,
16983            reviews: Vec::new(),
16984            e2e: Vec::new(),
16985            fix: None,
16986            blocking: 0,
16987            answered: 0,
16988            expected: 0,
16989            clean: true,
16990            verify_retried: false,
16991            e2e_deferred: false,
16992            e2e_defer_reason: None,
16993            progressed: false,
16994            vote_split: false,
16995            reconsideration: Vec::new(),
16996            verdict: None,
16997        }];
16998
16999        let mut runner = Runner {
17000            state,
17001            roles: ResolvedRoles {
17002                implementers: Vec::new(),
17003                judges: Vec::new(),
17004                reviewers: Vec::new(),
17005                fixer: None,
17006                conductor: conductor(),
17007                implementer_roster: Vec::new(),
17008                judge_roster: Vec::new(),
17009                reviewer_roster: Vec::new(),
17010            },
17011            sem: Arc::new(Semaphore::new(1)),
17012            pause: Pause::new(),
17013            interrupt: Pause::new(),
17014        };
17015
17016        runner.gate().await.expect("gate");
17017        assert!(
17018            runner.state.gate_ran,
17019            "zero configured commands is still a real attempt, not an unrun gate"
17020        );
17021        assert!(runner.state.gate.is_empty());
17022        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17023        assert_ne!(
17024            runner.state.status,
17025            RunStatus::Blocked,
17026            "a gate with nothing to check must not read as failed"
17027        );
17028
17029        runner.merge().await.expect("merge");
17030        assert_eq!(
17031            runner.state.status,
17032            RunStatus::Ready,
17033            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17034        );
17035    }
17036
17037    /// `Config::cache_dir` is derived from `verify.e2e` as well as
17038    /// `verify.gate` (so the e2e leg and the final gate never build against
17039    /// different directories). With zero `verify.gate` commands but a
17040    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
17041    /// that lease before discovering it had nothing to run - so a repo with
17042    /// no gate commands could come back `resource_blocked` (and therefore
17043    /// still `gate_ran == false`) on nothing but an unrelated run holding the
17044    /// cache, exactly the contention this run's own zero commands could
17045    /// never have touched. `gate` must recognise there is nothing to check
17046    /// before it ever asks for the lease.
17047    #[tokio::test]
17048    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17049        crate::run::pin_test_home();
17050        let home = crate::run::home();
17051
17052        let tmp = tempfile::tempdir().expect("tempdir");
17053        let repo = tmp.path().join("repo");
17054        std::fs::create_dir_all(&repo).unwrap();
17055        init_repo(&repo);
17056        // Unique to this test, so holding its lease cannot collide with
17057        // another test sharing the same process-wide `home`.
17058        let cache_dir = tmp.path().join("target");
17059
17060        let mut config = Config::default();
17061        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17062        // `verify.gate` stays empty (the default). Bounded so a regression
17063        // that does start waiting fails the test in seconds, not hangs it.
17064        config.graph.timeout_verify = Some(2);
17065        config.graph.timeout_resource_wait = 2;
17066
17067        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17068        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17069            .expect("no io error acquiring directly")
17070        {
17071            crate::cache::AcquireOutcome::Acquired(g) => g,
17072            crate::cache::AcquireOutcome::Busy(b) => {
17073                panic!("expected the direct acquire to win the lease first: {b:?}")
17074            }
17075        };
17076
17077        let mut state = RunState::new(
17078            repo.clone(),
17079            "main".to_owned(),
17080            "deadbeef".to_owned(),
17081            "task".to_owned(),
17082            config,
17083        );
17084        state.candidates = vec![Candidate {
17085            index: 0,
17086            label: 'A',
17087            agent: "alpha".to_owned(),
17088            branch: "does-not-exist".to_owned(),
17089            worktree: repo.clone(),
17090            summary: String::new(),
17091            stat: String::new(),
17092            files: 0,
17093            commits: 0,
17094            empty: false,
17095            failed: None,
17096            verified_noop: None,
17097            duration_ms: 0,
17098            folded: false,
17099        }];
17100        state.tally = Some(Tally {
17101            first_choice: BTreeMap::from([('A', 1)]),
17102            borda: BTreeMap::new(),
17103            winner: 'A',
17104            rankings: 1,
17105            unanimous_initial: true,
17106            deliberated: false,
17107            changed_votes: 0,
17108            unanimous_final: true,
17109            tie_break: None,
17110            judges: 0,
17111            present: 0,
17112            quorum: 0,
17113            met_quorum: true,
17114            uncontested: Some("only candidate A produced a change".to_owned()),
17115        });
17116        state.reviews = vec![ReviewRound {
17117            round: 1,
17118            head: "deadbeef".to_owned(),
17119            verified_head: None,
17120            verified_at: None,
17121            reviews: Vec::new(),
17122            e2e: Vec::new(),
17123            fix: None,
17124            blocking: 0,
17125            answered: 0,
17126            expected: 0,
17127            clean: true,
17128            verify_retried: false,
17129            e2e_deferred: false,
17130            e2e_defer_reason: None,
17131            progressed: false,
17132            vote_split: false,
17133            reconsideration: Vec::new(),
17134            verdict: None,
17135        }];
17136
17137        let mut runner = Runner {
17138            state,
17139            roles: ResolvedRoles {
17140                implementers: Vec::new(),
17141                judges: Vec::new(),
17142                reviewers: Vec::new(),
17143                fixer: None,
17144                conductor: conductor(),
17145                implementer_roster: Vec::new(),
17146                judge_roster: Vec::new(),
17147                reviewer_roster: Vec::new(),
17148            },
17149            sem: Arc::new(Semaphore::new(1)),
17150            pause: Pause::new(),
17151            interrupt: Pause::new(),
17152        };
17153
17154        let started = std::time::Instant::now();
17155        runner.gate().await.expect("gate");
17156        assert!(
17157            started.elapsed() < Duration::from_secs(1),
17158            "a gate with nothing to run must never wait on a lease it never needed"
17159        );
17160        assert!(
17161            runner.state.gate_ran,
17162            "zero commands is still a real, immediate attempt"
17163        );
17164        assert!(runner.state.gate.is_empty());
17165        assert_ne!(
17166            runner.state.status,
17167            RunStatus::Blocked,
17168            "must not read as resource-blocked on a lease it never asked for"
17169        );
17170    }
17171
17172    /// A separate process carries a live worker's environment throughout all
17173    /// verification entry points; the parallel harness never changes its env.
17174    #[cfg(unix)]
17175    #[tokio::test]
17176    async fn verification_is_standalone_inside_a_live_worker() {
17177        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17178        if std::env::var_os(CHILD).is_none() {
17179            let home = tempfile::tempdir().unwrap();
17180            let generation = crate::worker::pin_current(home.path()).unwrap();
17181            let output = std::process::Command::new(std::env::current_exe().unwrap())
17182                .args([
17183                    "--exact",
17184                    "graph::tests::verification_is_standalone_inside_a_live_worker",
17185                    "--nocapture",
17186                ])
17187                .env(CHILD, "1")
17188                .env("MAGI_HOME", home.path())
17189                .env("MAGI_WORKER_HOME", home.path())
17190                .env("MAGI_WORKER_ID", "abcdef")
17191                .env("MAGI_WORKER_GENERATION", &generation.id)
17192                .env("MAGI_WORKER_EXE", &generation.executable)
17193                .env("MAGI_RUN", "verification-parent")
17194                .env("MAGI_NODE", "gate")
17195                .env("MAGI_SEAT", "worker-fixture")
17196                .env("MAGI_PROCESS_OWNER", "fixture-owner")
17197                .env("MAGI_NO_AUTOUPDATE", "0")
17198                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17199                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17200                .env(
17201                    "VERIFY_EXPECT_TOOLCHAIN",
17202                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17203                )
17204                .env(
17205                    "VERIFY_EXPECT_CACHE",
17206                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17207                )
17208                .output()
17209                .unwrap();
17210            assert!(
17211                output.status.success()
17212                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17213                "{}\n{}",
17214                String::from_utf8_lossy(&output.stdout),
17215                String::from_utf8_lossy(&output.stderr)
17216            );
17217            return;
17218        }
17219        let home = crate::run::home();
17220        let directory = home.join("workers/jobs/abcdef");
17221        std::fs::create_dir_all(&directory).unwrap();
17222        let job = crate::worker::Job {
17223            id: "abcdef".into(),
17224            sequence: 1,
17225            kind: "task".into(),
17226            key: "task:verification".into(),
17227            payload: serde_json::json!({}),
17228            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17229            version: Some(env!("CARGO_PKG_VERSION").into()),
17230            status: crate::worker::Status::Running,
17231            result: None,
17232            pid: Some(std::process::id()),
17233            created_at: 0,
17234            finished_at: None,
17235        };
17236        std::fs::write(
17237            directory.join("job.json"),
17238            serde_json::to_vec(&job).unwrap(),
17239        )
17240        .unwrap();
17241        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17242        let assert_worker = || {
17243            assert_eq!(
17244                crate::worker::current_job(&home).unwrap().unwrap().id,
17245                job.id
17246            );
17247            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17248            assert_eq!(
17249                crate::worker::helper_executable(&home).unwrap(),
17250                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17251            );
17252            assert!(
17253                crate::updater::Checker::new(&crate::config::Update {
17254                    mode: crate::config::UpdateMode::Notify,
17255                    interval: None,
17256                    auto_restart: false,
17257                })
17258                .is_none(),
17259                "the driver remains pinned, even while tests are standalone"
17260            );
17261        };
17262        assert_worker();
17263        let repo = home.join("repo");
17264        std::fs::create_dir_all(&repo).unwrap();
17265        init_repo(&repo);
17266        // Keep storage, seat/control context, containment and tool lookup.
17267        // No inherited values (or credentials) are printed on success.
17268        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17269            test -z "${MAGI_WORKER_GENERATION+x}" &&
17270            test -z "${MAGI_WORKER_HOME+x}" &&
17271            test -z "${MAGI_WORKER_EXE+x}" &&
17272            test "$MAGI_HOME/repo" -ef . &&
17273            test "$MAGI_RUN" = verification-parent &&
17274            test "$MAGI_NODE" = gate &&
17275            test "$MAGI_SEAT" = worker-fixture &&
17276            test -n "$MAGI_PROCESS_OWNER" &&
17277            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17278            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17279            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17280            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17281        // Negative control: an ordinary child inherits the contamination.
17282        assert!(
17283            !std::process::Command::new("sh")
17284                .args(["-c", probe])
17285                .current_dir(&repo)
17286                .status()
17287                .unwrap()
17288                .success()
17289        );
17290        let unrelated_home = tempfile::tempdir().unwrap();
17291        assert!(
17292            crate::worker::current_job(unrelated_home.path()).is_err(),
17293            "an inherited worker id cannot be resolved in a unit fixture's home"
17294        );
17295        let mut runner = runner_at(RunStatus::Gating);
17296        runner.state.repo = repo.clone();
17297        runner.state.config.disk.min_free_bytes = 0;
17298        runner.state.config.coordination = crate::config::test_coordination();
17299        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17300        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17301        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17302        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17303        let winner = Candidate {
17304            index: 0,
17305            label: 'A',
17306            agent: "fixture".into(),
17307            branch: "main".into(),
17308            worktree: repo.clone(),
17309            summary: String::new(),
17310            stat: String::new(),
17311            files: 0,
17312            commits: 0,
17313            empty: false,
17314            failed: None,
17315            verified_noop: None,
17316            duration_ms: 0,
17317            folded: false,
17318        };
17319        let (e2e, retried, pids) = run_e2e_with_retry(
17320            &mut runner.state,
17321            &["sh".into(), "-c".into()],
17322            &[format!("{probe} && printf initial")],
17323            &repo,
17324            Duration::from_secs(10),
17325            "initial",
17326        )
17327        .await;
17328        assert!(!retried && pids.is_empty());
17329        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17330        assert_eq!(e2e[0].output_tail, "initial");
17331        let gate = runner.run_gate(&winner).await.unwrap();
17332        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17333        // An interrupted repair must recheck e2e from saved state. Landing
17334        // reuses that state and must run both pre-gate and gate again.
17335        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17336            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17337            passed: false,
17338            outcomes: Vec::new(),
17339        });
17340        runner.state.save().unwrap();
17341        let mut resumed = RunState::load(&runner.state.id).unwrap();
17342        let wait = crate::repo_lock::WaitPolicy::default();
17343        assert!(matches!(
17344            check_gate_repair_e2e(&mut resumed, &winner, &wait)
17345                .await
17346                .unwrap(),
17347            GateFix::Retry
17348        ));
17349        let saved = RunState::load(&resumed.id).unwrap();
17350        let repair = saved.gate_repair_e2e.unwrap();
17351        assert!(repair.passed);
17352        assert_eq!(repair.outcomes.len(), 1);
17353        assert!(repair.outcomes[0].ok());
17354        let landing = run_landing_gate(&mut resumed, &winner, &wait)
17355            .await
17356            .unwrap();
17357        assert_eq!(landing.len(), 1);
17358        assert!(landing[0].ok(), "{landing:?}");
17359        assert_eq!(
17360            std::fs::read_to_string(repo.join("phases")).unwrap(),
17361            "pre\ngate\ne2e\npre\ngate\n"
17362        );
17363        assert_worker();
17364    }
17365
17366    #[cfg(unix)]
17367    #[tokio::test]
17368    async fn verification_keeps_the_panic_and_the_full_output() {
17369        crate::run::pin_test_home();
17370        let tmp = tempfile::tempdir().unwrap();
17371        let mut config = Config::default();
17372        config.disk.min_free_bytes = 0;
17373        let mut state = RunState::new(
17374            tmp.path().to_owned(),
17375            "main".into(),
17376            "base".into(),
17377            "verification evidence".into(),
17378            config,
17379        );
17380        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
17381                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
17382                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
17383        let mut names = Vec::new();
17384        for _ in 0..2 {
17385            let (outcomes, _) = run_commands(
17386                &mut state,
17387                "gate",
17388                "gate",
17389                0,
17390                &["sh".into(), "-c".into()],
17391                &[script.into()],
17392                tmp.path(),
17393                Duration::from_secs(60),
17394            )
17395            .await;
17396            let tail = &outcomes[0].output_tail;
17397            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
17398            assert!(tail.contains("boom assertion"));
17399            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
17400            let line = tail.lines().next().unwrap();
17401            names.push(line.to_owned());
17402        }
17403        assert_ne!(names[0], names[1], "the second run must not overwrite");
17404        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
17405        assert!(full.len() > 500_000 && full.contains("boom assertion"));
17406        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
17407
17408        // Storage failures are evidence too; preserve the command's verdict.
17409        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
17410        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
17411        let (outcomes, _) = run_commands(
17412            &mut state,
17413            "gate",
17414            "gate",
17415            0,
17416            &["sh".into(), "-c".into()],
17417            &["echo failure-output; exit 7".into()],
17418            tmp.path(),
17419            Duration::from_secs(10),
17420        )
17421        .await;
17422        assert_eq!(outcomes[0].code, Some(7));
17423        assert!(state.events.iter().any(|e| {
17424            e.node == "gate" && e.message.contains("could not save the full gate output:")
17425        }));
17426    }
17427
17428    #[cfg(unix)]
17429    #[tokio::test]
17430    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
17431        crate::run::pin_test_home();
17432        let tmp = tempfile::tempdir().unwrap();
17433        let mut config = Config::default();
17434        config.disk.min_free_bytes = 0;
17435        let mut state = RunState::new(
17436            tmp.path().to_owned(),
17437            "main".into(),
17438            "base".into(),
17439            "verification evidence".into(),
17440            config,
17441        );
17442        let (outcomes, pids) = run_commands(
17443            &mut state,
17444            "gate",
17445            "gate",
17446            1,
17447            &["sh".into(), "-c".into()],
17448            &[
17449                "printf before-timeout; exec sleep 30".into(),
17450                "kill -TERM $$".into(),
17451                "exit 7".into(),
17452                "printf success".into(),
17453            ],
17454            tmp.path(),
17455            Duration::from_secs(1),
17456        )
17457        .await;
17458        wait_for_timed_out_children_to_die(&pids).await;
17459        assert_eq!(
17460            pids.len(),
17461            1,
17462            "unleased timeout retains its death-check PID"
17463        );
17464        assert_eq!(outcomes.len(), 4);
17465        assert_eq!(outcomes[0].code, None);
17466        assert!(outcomes[0].output_tail.contains("before-timeout"));
17467        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
17468        assert_eq!(outcomes[1].code, None);
17469        assert!(outcomes[1].output_tail.contains("command terminated:"));
17470        assert!(!outcomes[1].output_tail.contains("timed out"));
17471        assert_eq!(outcomes[2].code, Some(7));
17472        assert!(!outcomes[2].output_tail.contains("timed out"));
17473        assert_eq!(outcomes[3].code, Some(0));
17474        assert_eq!(outcomes[3].output_tail, "success");
17475        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
17476    }
17477
17478    /// The addendum's second gap: a `verify.gate` command running for real
17479    /// wall-clock time had nothing at all to show for it in `active` before
17480    /// `run_commands` learned to record it — a run could sit in `Gating` for
17481    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
17482    /// what was actually happening. Proven with a genuinely still-running
17483    /// command, not just a before/after check on the final state: a poller
17484    /// task reads the same `run.json` `gate()` is writing, the same way the
17485    /// phone or `magi show` would, while the shell command is still blocked
17486    /// on its own release marker.
17487    #[tokio::test]
17488    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
17489        crate::run::pin_test_home();
17490
17491        let tmp = tempfile::tempdir().expect("tempdir");
17492        let repo = tmp.path().join("repo");
17493        std::fs::create_dir_all(&repo).unwrap();
17494        init_repo(&repo);
17495
17496        let mut config = Config::default();
17497        config.verify.gate = vec![
17498            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
17499             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
17500                .to_owned(),
17501        ];
17502
17503        let mut state = RunState::new(
17504            repo.clone(),
17505            "main".to_owned(),
17506            "deadbeef".to_owned(),
17507            "task".to_owned(),
17508            config,
17509        );
17510        let run_id = state.id.clone();
17511        state.candidates = vec![Candidate {
17512            index: 0,
17513            label: 'A',
17514            agent: "alpha".to_owned(),
17515            branch: "does-not-exist".to_owned(),
17516            worktree: repo.clone(),
17517            summary: String::new(),
17518            stat: String::new(),
17519            files: 0,
17520            commits: 0,
17521            empty: false,
17522            failed: None,
17523            verified_noop: None,
17524            duration_ms: 0,
17525            folded: false,
17526        }];
17527        state.tally = Some(Tally {
17528            first_choice: BTreeMap::from([('A', 1)]),
17529            borda: BTreeMap::new(),
17530            winner: 'A',
17531            rankings: 1,
17532            unanimous_initial: true,
17533            deliberated: false,
17534            changed_votes: 0,
17535            unanimous_final: true,
17536            tie_break: None,
17537            judges: 0,
17538            present: 0,
17539            quorum: 0,
17540            met_quorum: true,
17541            uncontested: Some("only candidate A produced a change".to_owned()),
17542        });
17543        state.reviews = vec![ReviewRound {
17544            round: 1,
17545            head: "deadbeef".to_owned(),
17546            verified_head: None,
17547            verified_at: None,
17548            reviews: Vec::new(),
17549            e2e: Vec::new(),
17550            fix: None,
17551            blocking: 0,
17552            answered: 0,
17553            expected: 0,
17554            clean: true,
17555            verify_retried: false,
17556            e2e_deferred: false,
17557            e2e_defer_reason: None,
17558            progressed: false,
17559            vote_split: false,
17560            reconsideration: Vec::new(),
17561            verdict: None,
17562        }];
17563
17564        let mut runner = Runner {
17565            state,
17566            roles: ResolvedRoles {
17567                implementers: Vec::new(),
17568                judges: Vec::new(),
17569                reviewers: Vec::new(),
17570                fixer: None,
17571                conductor: conductor(),
17572                implementer_roster: Vec::new(),
17573                judge_roster: Vec::new(),
17574                reviewer_roster: Vec::new(),
17575            },
17576            sem: Arc::new(Semaphore::new(1)),
17577            pause: Pause::new(),
17578            interrupt: Pause::new(),
17579        };
17580
17581        let started_marker = repo.join("started.marker");
17582        let release_marker = repo.join("release.marker");
17583        let poller = tokio::spawn(async move {
17584            // Bounded so a regression that never records the task entry
17585            // fails this test in seconds instead of hanging the suite —
17586            // the same shape `a_park_requested_while_a_seat_is_mid_call_
17587            // does_not_cut_it_short` uses for the same reason.
17588            for _ in 0..100 {
17589                if started_marker.exists()
17590                    && let Ok(s) = crate::run::RunState::load(&run_id)
17591                    && let Some(a) = s.active.get("gate")
17592                {
17593                    std::fs::write(&release_marker, b"go").expect("release marker");
17594                    return Some(a.clone());
17595                }
17596                tokio::time::sleep(Duration::from_millis(50)).await;
17597            }
17598            None
17599        });
17600
17601        runner.gate().await.expect("gate");
17602        let captured = poller.await.expect("poller task");
17603        let captured = captured.expect(
17604            "the poller never saw a `gate` task entry in run.json while the command was \
17605             still blocked on its own release marker",
17606        );
17607
17608        assert_eq!(captured.task.as_deref(), Some("gate"));
17609        assert_eq!(captured.node, "gate");
17610        assert_eq!(captured.index, Some(1));
17611        assert_eq!(captured.total, Some(1));
17612        assert!(
17613            captured
17614                .command
17615                .as_deref()
17616                .is_some_and(|c| c.contains("started.marker")),
17617            "{captured:?}"
17618        );
17619
17620        assert!(
17621            runner.state.active.is_empty(),
17622            "the entry must be cleared once the command actually finished: {:?}",
17623            runner.state.active
17624        );
17625        assert!(runner.state.gate_ran);
17626        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
17627    }
17628
17629    /// The hand-off over a blocking finding a reviewer rejected on leaves a
17630    /// record for `land`; one with only a Minor, or no reject, leaves none.
17631    #[tokio::test]
17632    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
17633        use crate::verdict::{Finding, ReviewVote, Severity};
17634        crate::run::pin_test_home();
17635        let tmp = tempfile::tempdir().expect("tempdir");
17636        let repo = tmp.path().join("repo");
17637        std::fs::create_dir_all(&repo).unwrap();
17638        init_repo(&repo);
17639
17640        for (severity, vote, expect) in [
17641            (Severity::Major, ReviewVote::Reject, true),
17642            (Severity::Minor, ReviewVote::Reject, false),
17643            (Severity::Major, ReviewVote::Approve, false),
17644        ] {
17645            let mut round = review_round(false, 1, 1, 1, false, true);
17646            round.reviews = vec![ReviewRecord {
17647                reviewer: 1,
17648                agent: "alpha".to_owned(),
17649                summary: String::new(),
17650                findings: vec![Finding {
17651                    id: "R1-1-1".to_owned(),
17652                    severity,
17653                    file: None,
17654                    line: None,
17655                    title: "t".to_owned(),
17656                    detail: String::new(),
17657                }],
17658                vote: Some(vote),
17659                failed: None,
17660                duration_ms: 0,
17661                attempts: 0,
17662            }];
17663            let mut state = RunState::new(
17664                repo.clone(),
17665                "main".to_owned(),
17666                "deadbeef".to_owned(),
17667                "task".to_owned(),
17668                Config::default(),
17669            );
17670            state.reviews = vec![round];
17671            let mut runner = Runner {
17672                state,
17673                roles: ResolvedRoles {
17674                    implementers: Vec::new(),
17675                    judges: Vec::new(),
17676                    reviewers: Vec::new(),
17677                    fixer: None,
17678                    conductor: conductor(),
17679                    implementer_roster: Vec::new(),
17680                    judge_roster: Vec::new(),
17681                    reviewer_roster: Vec::new(),
17682                },
17683                sem: Arc::new(Semaphore::new(1)),
17684                pause: Pause::new(),
17685                interrupt: Pause::new(),
17686            };
17687            let shell = runner.state.config.shell();
17688            runner
17689                .stop_reviewing("round budget spent", &shell, &repo)
17690                .await
17691                .expect("stop_reviewing");
17692            assert_eq!(runner.state.status, RunStatus::Gating);
17693            assert_eq!(
17694                runner.state.contested_handoff.is_some(),
17695                expect,
17696                "{severity:?} + {vote:?}"
17697            );
17698        }
17699    }
17700
17701    /// The shape the incident this whole fix responds to actually had: the
17702    /// round budget spent, the last round's own e2e blocked on the shared
17703    /// build cache (held here by a live pid — this test process — exactly
17704    /// `cache`'s own unit tests' pattern for "another owner, still alive"
17705    /// without forking a process). `stop_reviewing` must retry it — not
17706    /// silently leave the round looking untouched (the catch-up-only half of
17707    /// the bug), and not read the contention as a red `e2e` and block the
17708    /// run on it (the other half). Called directly, the same way
17709    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
17710    /// above exercises `gate`, so this never needs a real cargo build to
17711    /// reach: the lease is never released, so `with_cache_lease` never gets
17712    /// past acquiring it into anything that would need a real workspace.
17713    #[tokio::test]
17714    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
17715        crate::run::pin_test_home();
17716        let home = crate::run::home();
17717
17718        let tmp = tempfile::tempdir().expect("tempdir");
17719        let repo = tmp.path().join("repo");
17720        std::fs::create_dir_all(&repo).unwrap();
17721        init_repo(&repo);
17722        let head = crate::git::rev_parse(&repo, "HEAD")
17723            .await
17724            .expect("rev-parse");
17725        // Unique to this test, so holding its lease cannot collide with
17726        // another test sharing the same process-wide `home`.
17727        let cache_dir = tmp.path().join("target");
17728
17729        let mut config = Config::default();
17730        config.verify.e2e = vec![format!(
17731            "CARGO_TARGET_DIR='{}' test -f README.md",
17732            cache_dir.display()
17733        )];
17734        config.graph.review_rounds = 1;
17735        // Bounded so a regression that does start waiting fails the test in
17736        // seconds, not hangs it.
17737        config.graph.timeout_verify = Some(2);
17738        config.graph.timeout_resource_wait = 2;
17739
17740        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17741        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17742            .expect("no io error acquiring directly")
17743        {
17744            crate::cache::AcquireOutcome::Acquired(g) => g,
17745            crate::cache::AcquireOutcome::Busy(b) => {
17746                panic!("expected the direct acquire to win the lease first: {b:?}")
17747            }
17748        };
17749
17750        let mut state = RunState::new(
17751            repo.clone(),
17752            "main".to_owned(),
17753            head.clone(),
17754            "task".to_owned(),
17755            config,
17756        );
17757        state.candidates = vec![Candidate {
17758            index: 0,
17759            label: 'A',
17760            agent: "alpha".to_owned(),
17761            branch: "does-not-exist".to_owned(),
17762            worktree: repo.clone(),
17763            summary: String::new(),
17764            stat: String::new(),
17765            files: 0,
17766            commits: 0,
17767            empty: false,
17768            failed: None,
17769            verified_noop: None,
17770            duration_ms: 0,
17771            folded: false,
17772        }];
17773        state.tally = Some(Tally {
17774            first_choice: BTreeMap::from([('A', 1)]),
17775            borda: BTreeMap::new(),
17776            winner: 'A',
17777            rankings: 1,
17778            unanimous_initial: true,
17779            deliberated: false,
17780            changed_votes: 0,
17781            unanimous_final: true,
17782            tie_break: None,
17783            judges: 0,
17784            present: 0,
17785            quorum: 0,
17786            met_quorum: true,
17787            uncontested: Some("only candidate A produced a change".to_owned()),
17788        });
17789        // The round budget's last round, deferred: `needs_catchup_run`'s
17790        // other trigger. `stop_reviewing`'s retry machinery must treat this
17791        // exactly like a resource-blocked attempt once it actually runs.
17792        state.reviews = vec![ReviewRound {
17793            round: 1,
17794            head: head.clone(),
17795            verified_head: None,
17796            verified_at: None,
17797            reviews: Vec::new(),
17798            e2e: Vec::new(),
17799            fix: None,
17800            blocking: 1,
17801            answered: 1,
17802            expected: 1,
17803            clean: false,
17804            verify_retried: false,
17805            e2e_deferred: true,
17806            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
17807            progressed: false,
17808            vote_split: false,
17809            reconsideration: Vec::new(),
17810            verdict: None,
17811        }];
17812
17813        let mut runner = Runner {
17814            state,
17815            roles: ResolvedRoles {
17816                implementers: Vec::new(),
17817                judges: Vec::new(),
17818                reviewers: Vec::new(),
17819                fixer: None,
17820                conductor: conductor(),
17821                implementer_roster: Vec::new(),
17822                judge_roster: Vec::new(),
17823                reviewer_roster: Vec::new(),
17824            },
17825            sem: Arc::new(Semaphore::new(1)),
17826            pause: Pause::new(),
17827            interrupt: Pause::new(),
17828        };
17829
17830        let shell = runner.state.config.shell();
17831        runner
17832            .stop_reviewing("round budget spent", &shell, &repo)
17833            .await
17834            .expect("stop_reviewing");
17835
17836        let last = runner.state.reviews.last().expect("round record");
17837        assert_eq!(
17838            last.e2e_status(),
17839            E2eStatus::ResourceBlocked,
17840            "the shared cache is still held; the attempt must read as blocked, not deferred or \
17841             failed: {last:?}"
17842        );
17843        assert_eq!(
17844            last.verified_head.as_deref(),
17845            Some(head.as_str()),
17846            "which commit this attempt targeted is known even though nothing finished checking \
17847             it"
17848        );
17849        let first_attempt_at = last
17850            .verified_at
17851            .expect("when this attempt ran is known too");
17852        assert_ne!(
17853            runner.state.status,
17854            RunStatus::Blocked,
17855            "contention is evidence about the machine, not the patch — it must not settle the \
17856             run as blocked: {:?}",
17857            runner.state.status
17858        );
17859        assert!(
17860            !runner
17861                .state
17862                .events
17863                .iter()
17864                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
17865            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
17866            runner.state.events
17867        );
17868
17869        // The cache is still held: a later reentry must retry the same
17870        // round's verification again — not leave it looking exactly as
17871        // untouched as the first blocked attempt, which is indistinguishable
17872        // from never having tried again at all.
17873        runner
17874            .stop_reviewing("round budget spent", &shell, &repo)
17875            .await
17876            .expect("stop_reviewing retry");
17877        assert_eq!(
17878            runner.state.reviews.len(),
17879            1,
17880            "no new round was started: {:?}",
17881            runner.state.reviews
17882        );
17883        let last = runner.state.reviews.last().expect("round record");
17884        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
17885        assert!(
17886            last.verified_at.expect("still known") > first_attempt_at,
17887            "a second reentry must be a fresh attempt, not a stale copy of the first"
17888        );
17889        assert_ne!(runner.state.status, RunStatus::Blocked);
17890
17891        held.release();
17892    }
17893
17894    #[cfg(unix)]
17895    #[tokio::test]
17896    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
17897        ask_test_home();
17898        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
17899            let tmp = tempfile::tempdir().unwrap();
17900            let repo = tmp.path().join("repo");
17901            std::fs::create_dir(&repo).unwrap();
17902            init_repo(&repo);
17903            let run_git = |args: &[&str]| {
17904                let out = std::process::Command::new("git")
17905                    .args(args)
17906                    .current_dir(&repo)
17907                    .output()
17908                    .unwrap();
17909                assert!(
17910                    out.status.success(),
17911                    "{args:?}: {}",
17912                    String::from_utf8_lossy(&out.stderr)
17913                );
17914            };
17915            let origin = tmp.path().join("origin.git");
17916            run_git(&["init", "--bare", origin.to_str().unwrap()]);
17917            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17918            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17919            run_git(&["add", "spec.rs"]);
17920            run_git(&["commit", "-m", "initial spec"]);
17921            run_git(&["push", "origin", "main"]);
17922            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17923            run_git(&["checkout", "-b", "fix/review"]);
17924            std::fs::write(
17925                repo.join("check.rs"),
17926                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17927            )
17928            .unwrap();
17929            run_git(&["add", "check.rs"]);
17930            run_git(&["commit", "-m", "use spec"]);
17931            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17932            run_git(&["checkout", "main"]);
17933            std::fs::write(
17934                repo.join("spec.rs"),
17935                "struct AgentSpec { max_turn_secs: u64 }\n",
17936            )
17937            .unwrap();
17938            run_git(&["add", "spec.rs"]);
17939            run_git(&["commit", "-m", "extend spec"]);
17940            run_git(&["push", "origin", "main"]);
17941            run_git(&["checkout", "fix/review"]);
17942
17943            let mut runner = runner_at(RunStatus::Reviewing);
17944            runner.state.repo = repo.clone();
17945            runner.state.base_commit = base;
17946            runner.state.pr_head = Some("fix/review".into());
17947            runner.state.config.graph.review_rounds = max_rounds;
17948            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17949            runner.state.config.graph.sessions = false;
17950            runner.state.config.disk.min_free_bytes = 0;
17951            runner.state.config.verify.e2e =
17952                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17953            runner.state.candidates = vec![Candidate {
17954                index: 0,
17955                label: 'A',
17956                agent: "fixture".into(),
17957                branch: "fix/review".into(),
17958                worktree: repo.clone(),
17959                summary: String::new(),
17960                stat: String::new(),
17961                files: 1,
17962                commits: 1,
17963                empty: false,
17964                failed: None,
17965                verified_noop: None,
17966                duration_ms: 0,
17967                folded: false,
17968            }];
17969            runner.state.tally = Some(Tally {
17970                first_choice: BTreeMap::new(),
17971                borda: BTreeMap::new(),
17972                winner: 'A',
17973                rankings: 0,
17974                unanimous_initial: true,
17975                deliberated: false,
17976                changed_votes: 0,
17977                unanimous_final: true,
17978                tie_break: None,
17979                judges: 0,
17980                present: 0,
17981                quorum: 0,
17982                met_quorum: true,
17983                uncontested: Some("review-only".into()),
17984            });
17985            let mut last = review_round(false, 0, 1, 1, false, false);
17986            last.head = reviewed.clone();
17987            last.e2e_deferred = true;
17988            last.e2e[0].resource_blocked = true;
17989            last.e2e[0].output_tail = "waiting for the shared build cache".into();
17990            if max_rounds == 1 {
17991                last.e2e.clear();
17992            }
17993            runner.state.reviews = vec![last];
17994            let calls = tmp.path().join("fix-calls");
17995            let mut fixer = spec("repair");
17996            fixer
17997                .env
17998                .insert("FIX_CALLS".into(), calls.display().to_string());
17999            fixer.command = vec!["sh".into(), "-c".into(), r#"
18000                echo fix >> "$FIX_CALLS"
18001                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18002                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18003                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18004                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18005            "#.into()];
18006            let mut reviewer = spec("reviewer");
18007            reviewer.command = vec![
18008                "sh".into(),
18009                "-c".into(),
18010                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18011                    .into(),
18012            ];
18013            runner.roles.fixer = Some(vec![fixer]);
18014            runner.roles.reviewers = vec![reviewer];
18015
18016            runner.sync_to_base().await.unwrap();
18017            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18018            assert_ne!(rebased, reviewed);
18019            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18020            if reentry {
18021                // Simulate a crash after catch-up evidence was saved but before
18022                // dispatching the fixer. No dedicated pending-repair field exists.
18023                let shell = runner.state.config.shell();
18024                runner
18025                    .stop_unresolved_review("deferred verification", &shell, &repo)
18026                    .await
18027                    .unwrap();
18028                assert!(runner.catchup_repairable());
18029                assert!(!calls.exists());
18030                runner.state = RunState::load(&runner.state.id).unwrap();
18031                assert!(runner.catchup_repairable());
18032            }
18033            runner.review_loop().await.unwrap();
18034            let first = &runner.state.reviews[0];
18035            assert_eq!(first.head, reviewed);
18036            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18037            assert!(
18038                first.verify_retried,
18039                "a reproduced build failure gets exactly one retry: {:?}",
18040                first.e2e
18041            );
18042            assert!(first.e2e[0].output_tail.contains("E0063"));
18043            assert!(!runner.catchup_repairable());
18044            if max_rounds == 2 {
18045                assert_eq!(
18046                    runner.state.status,
18047                    RunStatus::Gating,
18048                    "{}",
18049                    crate::report::run(&runner.state)
18050                );
18051                assert_eq!(runner.state.reviews.len(), 2);
18052                assert!(first.fix.as_ref().unwrap().committed);
18053                assert!(runner.state.reviews[1].clean);
18054                assert_eq!(
18055                    runner
18056                        .state
18057                        .reviews
18058                        .iter()
18059                        .filter(|r| r.fix.is_some())
18060                        .count(),
18061                    1
18062                );
18063                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18064                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18065            } else {
18066                assert_eq!(runner.state.status, RunStatus::Blocked);
18067                assert!(first.fix.is_none());
18068                assert!(!calls.exists());
18069                let recovery = crate::recovery::assess(
18070                    "prior",
18071                    Some(&runner.state),
18072                    &runner.landing_base(),
18073                    crate::recovery::Relation::Diverged,
18074                );
18075                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18076                let reasons = recovery.reasons.join("\n");
18077                assert!(
18078                    reasons.contains(&reviewed) && reasons.contains(&rebased),
18079                    "{reasons}"
18080                );
18081                assert!(
18082                    reasons.contains("base sync to")
18083                        && reasons.contains("additional review is required"),
18084                    "{reasons}"
18085                );
18086            }
18087        }
18088    }
18089
18090    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
18091    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
18092    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
18093    /// own top-of-function fast path (`review_conclusion`) correctly reads
18094    /// this shape as `None` rather than guessing `Blocked`, and the loop's
18095    /// own `for` range is empty once the round budget is spent, so
18096    /// `review_loop` must retry the check itself rather than silently doing
18097    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
18098    /// above exercises directly, but through `review_loop`'s own entry point
18099    /// this time, proving the wiring between the two rather than just the
18100    /// retry logic in isolation.
18101    #[tokio::test]
18102    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18103        crate::run::pin_test_home();
18104        let home = crate::run::home();
18105
18106        let tmp = tempfile::tempdir().expect("tempdir");
18107        let repo = tmp.path().join("repo");
18108        std::fs::create_dir_all(&repo).unwrap();
18109        init_repo(&repo);
18110        let head = crate::git::rev_parse(&repo, "HEAD")
18111            .await
18112            .expect("rev-parse");
18113        let cache_dir = tmp.path().join("target");
18114
18115        let mut config = Config::default();
18116        config.verify.e2e = vec![format!(
18117            "CARGO_TARGET_DIR='{}' test -f README.md",
18118            cache_dir.display()
18119        )];
18120        config.graph.review_rounds = 1;
18121        config.graph.timeout_verify = Some(2);
18122        config.graph.timeout_resource_wait = 2;
18123
18124        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18125        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18126            .expect("no io error acquiring directly")
18127        {
18128            crate::cache::AcquireOutcome::Acquired(g) => g,
18129            crate::cache::AcquireOutcome::Busy(b) => {
18130                panic!("expected the direct acquire to win the lease first: {b:?}")
18131            }
18132        };
18133
18134        let mut state = RunState::new(
18135            repo.clone(),
18136            "main".to_owned(),
18137            head.clone(),
18138            "task".to_owned(),
18139            config,
18140        );
18141        state.candidates = vec![Candidate {
18142            index: 0,
18143            label: 'A',
18144            agent: "alpha".to_owned(),
18145            branch: "does-not-exist".to_owned(),
18146            worktree: repo.clone(),
18147            summary: String::new(),
18148            stat: String::new(),
18149            files: 0,
18150            commits: 0,
18151            empty: false,
18152            failed: None,
18153            verified_noop: None,
18154            duration_ms: 0,
18155            folded: false,
18156        }];
18157        state.tally = Some(Tally {
18158            first_choice: BTreeMap::from([('A', 1)]),
18159            borda: BTreeMap::new(),
18160            winner: 'A',
18161            rankings: 1,
18162            unanimous_initial: true,
18163            deliberated: false,
18164            changed_votes: 0,
18165            unanimous_final: true,
18166            tie_break: None,
18167            judges: 0,
18168            present: 0,
18169            quorum: 0,
18170            met_quorum: true,
18171            uncontested: Some("only candidate A produced a change".to_owned()),
18172        });
18173        // The exact shape a prior process's `stop_reviewing` would have left
18174        // on disk: the round budget's last round, a real attempt already
18175        // made and already resource-blocked.
18176        state.reviews = vec![ReviewRound {
18177            round: 1,
18178            head: head.clone(),
18179            verified_head: Some(head.clone()),
18180            verified_at: Some(jiff::Timestamp::now()),
18181            reviews: Vec::new(),
18182            e2e: vec![CommandOutcome {
18183                command: format!(
18184                    "CARGO_TARGET_DIR='{}' test -f README.md",
18185                    cache_dir.display()
18186                ),
18187                code: None,
18188                output_tail: "waiting for the shared build cache".to_owned(),
18189                duration_ms: 0,
18190                resource_blocked: true,
18191            }],
18192            fix: None,
18193            blocking: 1,
18194            answered: 1,
18195            expected: 1,
18196            clean: false,
18197            verify_retried: false,
18198            e2e_deferred: false,
18199            e2e_defer_reason: None,
18200            progressed: false,
18201            vote_split: false,
18202            reconsideration: Vec::new(),
18203            verdict: None,
18204        }];
18205
18206        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18207        let mut runner = Runner {
18208            state,
18209            roles: ResolvedRoles {
18210                implementers: Vec::new(),
18211                judges: Vec::new(),
18212                reviewers: Vec::new(),
18213                fixer: None,
18214                conductor: conductor(),
18215                implementer_roster: Vec::new(),
18216                judge_roster: Vec::new(),
18217                reviewer_roster: Vec::new(),
18218            },
18219            sem: Arc::new(Semaphore::new(1)),
18220            pause: Pause::new(),
18221            interrupt: Pause::new(),
18222        };
18223
18224        // The lease is still held throughout, so this reentry's own retry is
18225        // also contended — proving `review_loop` actually tried again (not
18226        // that it happened to succeed) is what the timestamp comparison
18227        // below is for.
18228        runner.review_loop().await.expect("review_loop");
18229
18230        assert_eq!(
18231            runner.state.reviews.len(),
18232            1,
18233            "no new round was started on top of the unresolved one: {:?}",
18234            runner.state.reviews
18235        );
18236        let last = &runner.state.reviews[0];
18237        assert_eq!(
18238            last.e2e_status(),
18239            E2eStatus::ResourceBlocked,
18240            "still contended: {last:?}"
18241        );
18242        assert!(
18243            last.verified_at.expect("still known") > first_attempt_at,
18244            "review_loop must have actually retried the check, not left it exactly as found"
18245        );
18246        assert_ne!(
18247            runner.state.status,
18248            RunStatus::Blocked,
18249            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18250            runner.state.status
18251        );
18252
18253        held.release();
18254    }
18255
18256    #[tokio::test]
18257    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18258        crate::run::pin_test_home();
18259        for status in [RunStatus::Blocked, RunStatus::Gating] {
18260            let mut runner = runner_at(status);
18261            runner.state.config.graph.land = true;
18262            runner.state.gate_ran = true;
18263            runner.state.merge = Some(MergeOutcome {
18264                mode: MergeMode::Pr,
18265                ok: false,
18266                detail: "the fixer produced no commit".into(),
18267                empty: false,
18268            });
18269            let url = "https://example.invalid/o/r/pull/1";
18270            runner.state.pr = Some(crate::run::PrRecord {
18271                url: url.into(),
18272                number: 1,
18273                state: "open".into(),
18274                checks: "red".into(),
18275                round: 1,
18276                rounds: 1,
18277                red_at_merge: Vec::new(),
18278            });
18279            runner.state.save().unwrap();
18280            // The nonexistent repository makes observation fail before a forge
18281            // command can run. We still must reach land, rather than silently
18282            // return from the graph with its old gate and merge records.
18283            runner.execute_graph().await.unwrap();
18284            assert_eq!(runner.state.status, RunStatus::Blocked);
18285            assert!(runner.state.land_sync_pending);
18286            assert!(runner.state.events.iter().any(|event|
18287                event.node == "land" && event.message == format!("watching {url}")));
18288            assert_eq!(
18289                runner.state.merge.as_ref().unwrap().detail,
18290                "the fixer produced no commit"
18291            );
18292        }
18293    }
18294
18295    #[tokio::test]
18296    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18297        crate::run::pin_test_home();
18298        let tmp = tempfile::tempdir().expect("tempdir");
18299        let repo = tmp.path().join("repo");
18300        std::fs::create_dir_all(&repo).unwrap();
18301        init_repo(&repo);
18302
18303        let mut config = Config::default();
18304        config.merge.mode = MergeMode::Pr;
18305        config.graph.land = true;
18306        config.graph.land_approval = false;
18307
18308        let mut state = RunState::new(
18309            repo.clone(),
18310            "main".to_owned(),
18311            "deadbeef".to_owned(),
18312            "task".to_owned(),
18313            config,
18314        );
18315        state.candidates = vec![Candidate {
18316            index: 0,
18317            label: 'A',
18318            agent: "alpha".to_owned(),
18319            branch: "does-not-exist".to_owned(),
18320            worktree: repo.clone(),
18321            summary: String::new(),
18322            stat: String::new(),
18323            files: 0,
18324            commits: 0,
18325            empty: false,
18326            failed: None,
18327            verified_noop: None,
18328            duration_ms: 0,
18329            folded: false,
18330        }];
18331        state.tally = Some(Tally {
18332            first_choice: BTreeMap::from([('A', 1)]),
18333            borda: BTreeMap::new(),
18334            winner: 'A',
18335            rankings: 1,
18336            unanimous_initial: true,
18337            deliberated: false,
18338            changed_votes: 0,
18339            unanimous_final: true,
18340            tie_break: None,
18341            judges: 0,
18342            present: 0,
18343            quorum: 0,
18344            met_quorum: true,
18345            uncontested: Some("only candidate A produced a change".to_owned()),
18346        });
18347        state.reviews = vec![ReviewRound {
18348            round: 1,
18349            head: "deadbeef".to_owned(),
18350            verified_head: None,
18351            verified_at: None,
18352            reviews: Vec::new(),
18353            e2e: Vec::new(),
18354            fix: None,
18355            blocking: 0,
18356            answered: 0,
18357            expected: 0,
18358            clean: true,
18359            verify_retried: false,
18360            e2e_deferred: false,
18361            e2e_defer_reason: None,
18362            progressed: false,
18363            vote_split: false,
18364            reconsideration: Vec::new(),
18365            verdict: None,
18366        }];
18367        state.gate = vec![CommandOutcome {
18368            command: "test".to_owned(),
18369            code: Some(0),
18370            output_tail: String::new(),
18371            duration_ms: 0,
18372            resource_blocked: false,
18373        }];
18374        state.gate_ran = true;
18375        // A first pass through `merge` already pushed and opened this pull
18376        // request; `status` is `Landing` because a previous call into `land`
18377        // parked or was interrupted before it reached a terminal outcome.
18378        state.status = RunStatus::Landing;
18379        state.merge = Some(MergeOutcome {
18380            mode: MergeMode::Pr,
18381            ok: true,
18382            detail: "https://example.invalid/x/y/pull/1".to_owned(),
18383            empty: false,
18384        });
18385
18386        // The Landing-resume shortcut calls `run_land` directly rather than
18387        // through `merge`, which is exactly the call site that used to skip
18388        // `settle_questions` - see the fixture below.
18389        ask_test_home();
18390        // A real resume loads a persisted run; driver ownership is checked
18391        // against that record before the Landing shortcut is entered.
18392        state.save().expect("save landing run");
18393        let store = ask::Questions::open();
18394        let q = ask_open_question(&store, &state.id);
18395
18396        let mut runner = Runner {
18397            state,
18398            roles: ResolvedRoles {
18399                implementers: Vec::new(),
18400                judges: Vec::new(),
18401                reviewers: Vec::new(),
18402                fixer: None,
18403                conductor: conductor(),
18404                implementer_roster: Vec::new(),
18405                judge_roster: Vec::new(),
18406                reviewer_roster: Vec::new(),
18407            },
18408            sem: Arc::new(Semaphore::new(1)),
18409            pause: Pause::new(),
18410            interrupt: Pause::new(),
18411        };
18412
18413        // `execute`, not `merge` directly: the Landing-resume shortcut lives
18414        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
18415        // exactly because `review_loop` would otherwise clobber the marker
18416        // first.
18417        runner.execute().await.expect("execute");
18418
18419        assert_eq!(
18420            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
18421            Some("https://example.invalid/x/y/pull/1"),
18422            "reentry must not push again or open a second pull request over the \
18423             one `land` is already watching"
18424        );
18425        assert_ne!(
18426            runner.state.status,
18427            RunStatus::Landing,
18428            "land could not actually reach the fake pull request, so it must \
18429             have given up rather than left the run silently parked forever"
18430        );
18431        // `land` could not reach the fake pull request, so it gave up into
18432        // `Blocked` - still resumable, so the question must not have been
18433        // swept just because this branch now also calls `settle_questions`.
18434        assert_eq!(runner.state.status, RunStatus::Blocked);
18435        assert!(
18436            store.get(&q.id).unwrap().status.open(),
18437            "Blocked is still alive; settle_questions must have been a no-op here"
18438        );
18439    }
18440
18441    fn state_with_round(round: ReviewRound) -> RunState {
18442        let mut s = RunState::new(
18443            PathBuf::from("/repo"),
18444            "main".to_owned(),
18445            "abc1234".to_owned(),
18446            "add retries".to_owned(),
18447            Config::default(),
18448        );
18449        s.reviews = vec![round];
18450        s
18451    }
18452
18453    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
18454        crate::verdict::Finding {
18455            id: id.to_owned(),
18456            severity,
18457            file: None,
18458            line: None,
18459            title: title.to_owned(),
18460            detail: String::new(),
18461        }
18462    }
18463
18464    #[test]
18465    fn pr_body_names_open_findings_and_declined_ones() {
18466        let round = ReviewRound {
18467            round: 2,
18468            head: "deadbee".to_owned(),
18469            verified_head: None,
18470            verified_at: None,
18471            reviews: vec![ReviewRecord {
18472                attempts: 0,
18473                reviewer: 1,
18474                agent: "alpha".to_owned(),
18475                summary: String::new(),
18476                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
18477                vote: None,
18478                failed: None,
18479                duration_ms: 0,
18480            }],
18481            e2e: vec![CommandOutcome {
18482                command: "cargo test".to_owned(),
18483                code: Some(0),
18484                output_tail: String::new(),
18485                duration_ms: 0,
18486                resource_blocked: false,
18487            }],
18488            verify_retried: false,
18489            e2e_deferred: false,
18490            e2e_defer_reason: None,
18491            fix: Some(FixRecord {
18492                agent: "alpha".to_owned(),
18493                addressed: Vec::new(),
18494                rejected: vec![crate::verdict::Rejection {
18495                    id: "R1-1-1".to_owned(),
18496                    why: "not reachable from any caller".to_owned(),
18497                }],
18498                notes: String::new(),
18499                committed: true,
18500                failed: None,
18501                duration_ms: 0,
18502                continuation: None,
18503            }),
18504            blocking: 0,
18505            answered: 1,
18506            expected: 1,
18507            clean: false,
18508            progressed: true,
18509            vote_split: false,
18510            reconsideration: Vec::new(),
18511            verdict: None,
18512        };
18513        let state = state_with_round(round);
18514        let body = pr_message(&state, 'A').body;
18515
18516        assert!(body.contains("add retries"), "the task must still be there");
18517        assert!(body.contains("R2-1-1"), "{body}");
18518        assert!(!body.contains("unused import"), "{body}");
18519        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
18520        assert!(
18521            !body.contains("not reachable from any caller"),
18522            "the reason it was declined: {body}"
18523        );
18524    }
18525
18526    #[test]
18527    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
18528        let round = ReviewRound {
18529            round: 1,
18530            head: "deadbee".to_owned(),
18531            verified_head: None,
18532            verified_at: None,
18533            reviews: vec![ReviewRecord {
18534                attempts: 0,
18535                reviewer: 1,
18536                agent: "alpha".to_owned(),
18537                summary: String::new(),
18538                findings: Vec::new(),
18539                vote: None,
18540                failed: None,
18541                duration_ms: 0,
18542            }],
18543            e2e: Vec::new(),
18544            verify_retried: false,
18545            e2e_deferred: false,
18546            e2e_defer_reason: None,
18547            fix: None,
18548            blocking: 0,
18549            answered: 1,
18550            expected: 1,
18551            clean: true,
18552            progressed: false,
18553            vote_split: false,
18554            reconsideration: Vec::new(),
18555            verdict: None,
18556        };
18557        let state = state_with_round(round);
18558        let body = pr_message(&state, 'A').body;
18559        assert!(!body.contains("Open review findings"), "{body}");
18560        assert!(!body.contains("Declined"), "{body}");
18561    }
18562
18563    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
18564        let mut state = RunState::new(
18565            PathBuf::from("/repo"),
18566            "main".to_owned(),
18567            "abc1234".to_owned(),
18568            instruction.to_owned(),
18569            Config::default(),
18570        );
18571        state.candidates.push(Candidate {
18572            index: 0,
18573            label: 'A',
18574            agent: "alpha".to_owned(),
18575            branch: "magi/x/A".to_owned(),
18576            worktree: PathBuf::from("/wt"),
18577            summary: summary.to_owned(),
18578            stat: String::new(),
18579            files: 1,
18580            commits: 1,
18581            empty: false,
18582            failed: None,
18583            verified_noop: None,
18584            folded: false,
18585            duration_ms: 0,
18586        });
18587        state
18588    }
18589
18590    fn review_state(subjects: &[&str]) -> RunState {
18591        let mut state = state_with_summary(
18592            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
18593            "",
18594        );
18595        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18596        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
18597        state
18598    }
18599
18600    /// A branch rebased onto a main that moved past the recorded
18601    /// `base_commit` is titled from its own first commit, never main's.
18602    #[tokio::test]
18603    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
18604        ask_test_home();
18605        let tmp = tempfile::tempdir().unwrap();
18606        let repo = tmp.path().join("repo");
18607        std::fs::create_dir_all(&repo).unwrap();
18608        init_repo(&repo);
18609        let origin = tmp.path().join("origin.git");
18610        let g = |dir: &Path, args: &[&str]| {
18611            let out = std::process::Command::new("git")
18612                .args(args)
18613                .current_dir(dir)
18614                .quiet()
18615                .output()
18616                .expect("spawn git");
18617            assert!(
18618                out.status.success(),
18619                "git {args:?}: {}",
18620                String::from_utf8_lossy(&out.stderr)
18621            );
18622        };
18623        g(
18624            tmp.path(),
18625            &[
18626                "clone",
18627                "--bare",
18628                "-q",
18629                repo.to_str().unwrap(),
18630                origin.to_str().unwrap(),
18631            ],
18632        );
18633        g(
18634            &repo,
18635            &["remote", "add", "origin", origin.to_str().unwrap()],
18636        );
18637        let c1 = git::rev_parse(&repo, "main").await.unwrap();
18638
18639        // Main moves on; the branch is built on top of the new main.
18640        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
18641        g(&repo, &["add", "-A"]);
18642        g(
18643            &repo,
18644            &["commit", "-q", "-m", "chore(deps): update a crate"],
18645        );
18646        g(&repo, &["push", "-q", "origin", "main"]);
18647        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
18648        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
18649        g(&repo, &["add", "-A"]);
18650        g(
18651            &repo,
18652            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
18653        );
18654        g(&repo, &["checkout", "-q", "main"]);
18655
18656        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
18657        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
18658        // Without a readable tracking ref the recorded base's merge base is used.
18659        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
18660        assert_eq!(fallback, c1);
18661
18662        let mut state = review_state(&[
18663            "chore(deps): update a crate",
18664            "fix(daemon): apply a chosen action",
18665        ]);
18666        state.repo = repo.clone();
18667        state.base_branch = "main".to_owned();
18668        state.base_commit = c1;
18669        refresh_reviewed_commits(&mut state, "feat/own").await;
18670        assert_eq!(
18671            state.reviewed_commits,
18672            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
18673        );
18674        assert_eq!(
18675            review_title(&state).as_deref(),
18676            Some("fix(daemon): apply a chosen action")
18677        );
18678        assert_eq!(
18679            leaked_subjects(&state, "feat/own").await,
18680            Some(vec!["chore(deps): update a crate".to_owned()])
18681        );
18682        // A stale tracking ref is still used when the fetch fails, but the
18683        // leak list is withheld.
18684        state.config.merge.remote = "nowhere".to_owned();
18685        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
18686    }
18687
18688    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
18689    /// with `origin/main` one commit behind it, and a summarizing agent that
18690    /// prints whatever `reply` holds.
18691    struct SummaryFixture {
18692        _tmp: tempfile::TempDir,
18693        repo: PathBuf,
18694        reply: PathBuf,
18695        runner: Runner,
18696    }
18697
18698    fn summary_fixture() -> SummaryFixture {
18699        ask_test_home();
18700        let tmp = tempfile::tempdir().unwrap();
18701        let repo = tmp.path().join("repo");
18702        std::fs::create_dir_all(&repo).unwrap();
18703        init_repo(&repo);
18704        let origin = tmp.path().join("origin.git");
18705        let g = |args: &[&str]| {
18706            let out = std::process::Command::new("git")
18707                .args(args)
18708                .current_dir(&repo)
18709                .quiet()
18710                .output()
18711                .expect("spawn git");
18712            assert!(
18713                out.status.success(),
18714                "git {args:?}: {}",
18715                String::from_utf8_lossy(&out.stderr)
18716            );
18717        };
18718        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
18719        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
18720        g(&["push", "-q", "origin", "main"]);
18721        g(&["checkout", "-q", "-b", "feat/own"]);
18722        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
18723        g(&["add", "-A"]);
18724        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
18725        let reply = tmp.path().join("reply.txt");
18726        std::fs::write(
18727            &reply,
18728            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
18729        )
18730        .unwrap();
18731        let mut runner = runner_at(RunStatus::Gating);
18732        let mut state = review_state(&["feat(own): add the own file"]);
18733        state.repo = repo.clone();
18734        state.base_branch = "main".to_owned();
18735        state.candidates[0].branch = "feat/own".to_owned();
18736        state.candidates[0].worktree = repo.clone();
18737        let mut author = spec("alpha");
18738        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
18739        author
18740            .env
18741            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
18742        state.config.agents = vec![author];
18743        runner.state = state;
18744        SummaryFixture {
18745            _tmp: tmp,
18746            repo,
18747            reply,
18748            runner,
18749        }
18750    }
18751
18752    async fn posted(f: &mut SummaryFixture) -> PrMessage {
18753        let winner = f.runner.state.candidates[0].clone();
18754        f.runner
18755            .guarded_pr_message(&winner, None, true)
18756            .await
18757            .unwrap()
18758            .expect("an English description never parks")
18759    }
18760
18761    fn clean_round_with_minor_findings() -> ReviewRound {
18762        let mut round = passing_round(2, None);
18763        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
18764        round.reviews = vec![ReviewRecord {
18765            attempts: 0,
18766            reviewer: 1,
18767            agent: "alpha".into(),
18768            summary: "内部レビュー".into(),
18769            findings: vec![
18770                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
18771                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
18772            ],
18773            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
18774            failed: None,
18775            duration_ms: 0,
18776        }];
18777        round
18778    }
18779
18780    #[tokio::test]
18781    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
18782        for review_only in [false, true] {
18783            let mut f = summary_fixture();
18784            if !review_only {
18785                f.runner.state.candidates[0].agent = "alpha".into();
18786            }
18787            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
18788            assert!(f.runner.state.reviews[0].clean);
18789            assert!(f.runner.state.reviews[0].fix.is_none());
18790            assert!(f.runner.state.open_findings().is_empty());
18791            assert_eq!(
18792                crate::pr_outcome::required_finding_ids(&f.runner.state),
18793                ["R2-1-1", "R2-1-2"]
18794            );
18795            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18796            let evidence = crate::pr_outcome::summary_evidence(
18797                &f.runner.state,
18798                crate::pr_outcome::FinalHead::At(&head),
18799            );
18800            assert!(
18801                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
18802            );
18803            // The ordinary reply omits both IDs: no cached description is adopted.
18804            let omitted = posted(&mut f).await;
18805            assert!(omitted.body.contains("No description"));
18806            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
18807            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
18808            assert!(f.runner.state.change_summary.is_none());
18809            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
18810            let described = posted(&mut f).await;
18811            assert!(described.body.contains("improve documentation for callers"));
18812            assert!(described.body.contains("remaining edge case"));
18813            assert!(described.body.contains("## Open review findings"));
18814            assert!(!described.body.contains("未修正"));
18815        }
18816    }
18817
18818    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
18819        let mut f = summary_fixture();
18820        if !review_only {
18821            f.runner.state.candidates[0].agent = "alpha".into();
18822        }
18823        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18824        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
18825        fixed
18826            .fix
18827            .as_mut()
18828            .unwrap()
18829            .rejected
18830            .push(crate::verdict::Rejection {
18831                id: "R1-1-1".into(),
18832                why: "呼び出されないコードのため辞退".into(),
18833            });
18834        let mut clean = clean_round_with_minor_findings();
18835        clean.verified_head = Some(head.clone());
18836        f.runner.state.reviews = vec![fixed, clean];
18837        ok_gate(&mut f.runner.state, Some(&head));
18838        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
18839        f.runner.state.config.agents.push(scripted("beta", reply));
18840        f.runner.state.config.roles.language_judge =
18841            Some(crate::config::AgentChoice::One("beta".into()));
18842        f
18843    }
18844
18845    #[tokio::test]
18846    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
18847        for review_only in [false, true] {
18848            let mut f = rewrite_fixture_with_findings(review_only,
18849                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
18850            let m = posted(&mut f).await;
18851            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18852            let expected =
18853                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
18854            assert!(m.body.contains(&expected));
18855            assert!(m.body.contains("## Review rounds"));
18856            assert!(
18857                m.body
18858                    .contains("Gate (`verify.gate`): passed on the final commit")
18859            );
18860            assert!(
18861                m.body
18862                    .contains("Tests (`verify.e2e`): passed on the final commit")
18863            );
18864            let c = f.runner.state.change_summary.as_ref().unwrap();
18865            assert!(
18866                m.body
18867                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
18868            );
18869            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18870            assert!(crate::pr_outcome::covers_required_findings(
18871                &f.runner.state,
18872                &m.body
18873            ));
18874            assert!(m.body.contains("documentation remains incomplete"));
18875            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
18876            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
18877            assert!(f.runner.state.github_text.is_none());
18878        }
18879    }
18880
18881    #[tokio::test]
18882    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
18883        for review_only in [false, true] {
18884            for reply in [
18885                "TITLE: fix: public description\nA short English description.",
18886                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
18887            ] {
18888                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
18889                let winner = f.runner.state.candidates[0].clone();
18890                let posted = f
18891                    .runner
18892                    .guarded_pr_message(&winner, None, true)
18893                    .await
18894                    .unwrap();
18895                assert!(posted.is_none());
18896                assert!(f.runner.state.parked);
18897                let withheld = f.runner.state.github_text.as_ref().unwrap();
18898                assert!(
18899                    withheld.rewrite.is_none(),
18900                    "an incomplete rewrite must not be selectable"
18901                );
18902                let q = ask::Questions::open()
18903                    .get(withheld.question.as_ref().unwrap())
18904                    .unwrap();
18905                assert!(
18906                    !q.choices
18907                        .contains(&crate::github_text::USE_REWRITE.to_owned())
18908                );
18909                assert!(
18910                    q.detail.contains("R1-1-1")
18911                        && q.detail.contains("R2-1-1")
18912                        && q.detail.contains("R2-1-2")
18913                );
18914            }
18915        }
18916    }
18917
18918    #[tokio::test]
18919    async fn japanese_reports_stay_local_in_both_public_run_modes() {
18920        for review_only in [false, true] {
18921            let mut f = summary_fixture();
18922            if !review_only {
18923                f.runner.state.candidates[0].agent = "alpha".into();
18924            }
18925            f.runner.state.instruction = "内部タスクの説明".into();
18926            f.runner.state.candidates[0].summary = "内部の候補報告".into();
18927            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18928            f.runner.state.reviews = vec![
18929                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18930                round_with_notes(2, Some("二回目の内部修正報告です。")),
18931                passing_round(3, Some(&head)),
18932            ];
18933            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18934            ok_gate(&mut f.runner.state, Some(&head));
18935            let captured = f.repo.join("public-prompt.txt");
18936            let author = &mut f.runner.state.config.agents[0];
18937            author.command = vec![
18938                "sh".into(),
18939                "-c".into(),
18940                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18941            ];
18942            author.env.insert(
18943                "PROMPT_FILE".into(),
18944                captured.to_string_lossy().into_owned(),
18945            );
18946            let m = posted(&mut f).await;
18947            assert!(m.body.contains("Adds `own.txt`"));
18948            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18949            assert!(m.body.contains("R1-1-1"));
18950            assert!(
18951                m.body
18952                    .contains("Gate (`verify.gate`): passed on the final commit")
18953            );
18954            assert!(
18955                m.body
18956                    .contains("Tests (`verify.e2e`): passed on the final commit")
18957            );
18958            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18959            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18960            let prompt = std::fs::read_to_string(captured).unwrap();
18961            assert!(prompt.contains(&format!(
18962                "git diff {} {head}",
18963                f.runner.state.change_summary.as_ref().unwrap().base
18964            )));
18965            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18966            assert_eq!(
18967                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18968                "二回目の内部修正報告です。"
18969            );
18970        }
18971    }
18972
18973    #[tokio::test]
18974    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18975        for review_only in [false, true] {
18976            let mut f = summary_fixture();
18977            if !review_only {
18978                f.runner.state.candidates[0].agent = "alpha".into();
18979            }
18980            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18981            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18982            std::fs::write(&f.reply, "no usable description").unwrap();
18983            let failed = posted(&mut f).await;
18984            assert!(
18985                failed
18986                    .body
18987                    .contains("No description of what this change does could be generated")
18988            );
18989            assert!(failed.body.is_ascii());
18990            assert!(f.runner.state.github_text.is_none());
18991            assert!(f.runner.state.change_summary.is_none());
18992            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18993            let evidence = crate::pr_outcome::summary_evidence(
18994                &f.runner.state,
18995                crate::pr_outcome::FinalHead::At(&head),
18996            );
18997            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18998            assert!(
18999                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19000                    .exists()
19001            );
19002            f.runner.state =
19003                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19004            std::fs::write(
19005                &f.reply,
19006                "TITLE: feat(own): add the own file\nAdds the committed own file.",
19007            )
19008            .unwrap();
19009            let retried = posted(&mut f).await;
19010            assert!(retried.body.contains("Adds the committed own file."));
19011            assert_eq!(
19012                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19013                "日本語の修正報告"
19014            );
19015        }
19016    }
19017
19018    #[tokio::test]
19019    async fn same_head_verification_and_findings_invalidate_public_prose() {
19020        let mut f = summary_fixture();
19021        posted(&mut f).await;
19022        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19023        let original = f
19024            .runner
19025            .state
19026            .change_summary
19027            .as_ref()
19028            .unwrap()
19029            .evidence
19030            .clone();
19031        ok_gate(&mut f.runner.state, Some(&head));
19032        std::fs::write(
19033            &f.reply,
19034            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19035        )
19036        .unwrap();
19037        let verified = posted(&mut f).await;
19038        assert!(verified.body.contains("now verified"));
19039        assert_ne!(
19040            f.runner.state.change_summary.as_ref().unwrap().evidence,
19041            original
19042        );
19043        let mut round = round_with_notes(1, Some("内部報告"));
19044        round.reviews.push(ReviewRecord {
19045            attempts: 0,
19046            reviewer: 1,
19047            agent: "alpha".into(),
19048            summary: "内部レビュー".into(),
19049            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19050            vote: None,
19051            failed: None,
19052            duration_ms: 0,
19053        });
19054        round
19055            .fix
19056            .as_mut()
19057            .unwrap()
19058            .rejected
19059            .push(crate::verdict::Rejection {
19060                id: "R1-1-1".into(),
19061                why: "呼び出し元から到達できない".into(),
19062            });
19063        f.runner.state.reviews = vec![round];
19064        // A description omitting the required ID is unusable, even with a title.
19065        let omitted = posted(&mut f).await;
19066        assert!(omitted.body.contains("No description"));
19067        assert!(omitted.body.contains("R1-1-1"));
19068        assert!(!omitted.body.contains("到達"));
19069        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19070        let described = posted(&mut f).await;
19071        assert!(described.body.contains("unreachable from callers"));
19072        assert!(described.body.contains("R1-1-1"));
19073        assert!(!described.body.contains("到達"));
19074        let evidence = f
19075            .runner
19076            .state
19077            .change_summary
19078            .as_ref()
19079            .unwrap()
19080            .evidence
19081            .clone();
19082        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19083        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19084        assert!(posted(&mut f).await.body.contains("rationale updated"));
19085        assert_ne!(
19086            f.runner.state.change_summary.as_ref().unwrap().evidence,
19087            evidence
19088        );
19089        // Old serialized summaries have no evidence fingerprint and must regenerate.
19090        f.runner
19091            .state
19092            .change_summary
19093            .as_mut()
19094            .unwrap()
19095            .evidence
19096            .clear();
19097        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19098        assert!(posted(&mut f).await.body.contains("current specification"));
19099    }
19100
19101    #[tokio::test]
19102    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19103        let mut f = summary_fixture();
19104        let m = posted(&mut f).await;
19105        assert_eq!(m.title, "feat(own): add the own file");
19106        assert!(
19107            m.body
19108                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19109            "{}",
19110            m.body
19111        );
19112        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19113        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19114        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19115        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19116        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19117        let kept = f.runner.state.change_summary.as_ref().unwrap();
19118        assert_eq!(
19119            (kept.head.as_str(), kept.base.as_str()),
19120            (head.as_str(), base.as_str())
19121        );
19122        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19123    }
19124
19125    #[tokio::test]
19126    async fn review_only_summary_receives_the_configured_github_language() {
19127        let mut f = summary_fixture();
19128        f.runner.state.config.graph.github_language = "ja".into();
19129        let captured = f.repo.join("summary-prompt.txt");
19130        let author = &mut f.runner.state.config.agents[0];
19131        author.command = vec![
19132            "sh".into(),
19133            "-c".into(),
19134            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19135        ];
19136        author.env.insert(
19137            "PROMPT_FILE".into(),
19138            captured.to_string_lossy().into_owned(),
19139        );
19140        std::fs::write(
19141            &f.reply,
19142            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19143        )
19144        .unwrap();
19145        let m = posted(&mut f).await;
19146        let prompt = std::fs::read_to_string(captured).unwrap();
19147        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19148        assert!(!prompt.contains("in English"));
19149        assert_eq!(m.title, "feat(own): ファイルを追加");
19150        assert!(m.body.contains("own.txt を追加します。"));
19151    }
19152
19153    #[tokio::test]
19154    async fn review_fixes_come_after_the_whole_change_summary() {
19155        let mut f = summary_fixture();
19156        f.runner.state.reviews = vec![
19157            round_with_notes(1, Some("handled the PATH case")),
19158            round_with_notes(2, None),
19159        ];
19160        let m = posted(&mut f).await;
19161        let summary = m.body.find("## Summary").expect("summary");
19162        let fixes = m.body.find("## Review rounds").expect("review rounds");
19163        assert!(summary < fixes, "{}", m.body);
19164        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19165        assert!(m.body.contains("## Final verification"), "{}", m.body);
19166    }
19167
19168    #[tokio::test]
19169    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19170        let mut f = summary_fixture();
19171        let first = posted(&mut f).await;
19172        let first_head = f.runner.state.change_summary.clone().unwrap().head;
19173        // The same head reuses the record: a different reply is not read.
19174        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19175        let again = posted(&mut f).await;
19176        assert_eq!((again.title, again.body), (first.title, first.body));
19177
19178        let g = |args: &[&str]| {
19179            let out = std::process::Command::new("git")
19180                .args(args)
19181                .current_dir(&f.repo)
19182                .quiet()
19183                .output()
19184                .expect("spawn git");
19185            assert!(out.status.success(), "git {args:?}");
19186        };
19187        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19188        g(&["add", "-A"]);
19189        g(&["commit", "-q", "-m", "fix(own): handle more"]);
19190        let second = posted(&mut f).await;
19191        let kept = f.runner.state.change_summary.clone().unwrap();
19192        assert_ne!(kept.head, first_head);
19193        assert!(second.body.contains("Not read yet."), "{}", second.body);
19194        assert!(!second.body.contains("so the first summary is read"));
19195        assert_eq!(second.title, "feat(own): other");
19196    }
19197
19198    #[tokio::test]
19199    async fn an_unavailable_summary_is_stated_and_never_invented() {
19200        let mut f = summary_fixture();
19201        std::fs::write(&f.reply, "no title line here").unwrap();
19202        let m = posted(&mut f).await;
19203        assert!(
19204            m.body
19205                .contains("No description of what this change does could be generated"),
19206            "{}",
19207            m.body
19208        );
19209        assert!(m.body.contains("internal reports remain in local records"));
19210        assert!(f.runner.state.change_summary.is_none());
19211        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19212        // A preview asks nobody and claims nothing.
19213        let winner = f.runner.state.candidates[0].clone();
19214        let preview = f
19215            .runner
19216            .guarded_pr_message(&winner, None, false)
19217            .await
19218            .unwrap()
19219            .unwrap();
19220        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19221    }
19222
19223    #[test]
19224    fn pr_message_review_single_commit_uses_its_subject() {
19225        let state = review_state(&["feat(nats): per-role user"]);
19226        let m = pr_message(&state, 'A');
19227        assert_eq!(m.title, "feat(nats): per-role user");
19228        assert!(!m.body.contains("Review the work already"), "{}", m.body);
19229        assert!(m.body.contains("## Commits under review"), "{}", m.body);
19230    }
19231
19232    #[test]
19233    fn pr_message_review_multi_commit_takes_the_oldest() {
19234        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19235        let m = pr_message(&state, 'A');
19236        assert_eq!(m.title, "feat: the change");
19237        for s in ["feat: the change", "fix: typo", "fix: again"] {
19238            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19239        }
19240    }
19241
19242    #[test]
19243    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19244        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19245            let mut state = review_state(&[first, "fix: later fixup"]);
19246            state.candidates[0].branch = "機能/ログイン".to_owned();
19247            let m = pr_message(&state, 'A');
19248            assert!(
19249                m.title.starts_with("chore: land candidate A of run"),
19250                "{}",
19251                m.title
19252            );
19253        }
19254    }
19255
19256    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19257        BranchFacts {
19258            commits: commits
19259                .iter()
19260                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19261                .collect(),
19262            stat: stat.to_owned(),
19263        }
19264    }
19265
19266    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19267        let mut r = review_round(true, 0, 1, 1, true, true);
19268        r.round = round;
19269        r.fix = notes.map(|n| FixRecord {
19270            agent: "fixer".to_owned(),
19271            addressed: Vec::new(),
19272            rejected: Vec::new(),
19273            notes: n.to_owned(),
19274            committed: true,
19275            failed: None,
19276            duration_ms: 0,
19277            continuation: None,
19278        });
19279        r
19280    }
19281
19282    #[test]
19283    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19284        let state = review_state(&["ignored"]);
19285        let f = facts(
19286            &[
19287                (
19288                    "fix(login): resolve PATH on macOS",
19289                    "Login shells skip rc files.",
19290                ),
19291                ("fix: address review", ""),
19292            ],
19293            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19294        );
19295        let m = pr_message_with(&state, 'A', Some(&f));
19296        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19297        assert!(
19298            m.body
19299                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
19300            "{}",
19301            m.body
19302        );
19303        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19304        assert!(m.body.contains("## Diff stat"), "{}", m.body);
19305        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19306        for banned in [
19307            "Review the work already",
19308            "no task statement",
19309            "Original task",
19310        ] {
19311            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19312        }
19313        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19314    }
19315
19316    #[test]
19317    fn pr_message_review_truncates_a_huge_first_commit_body() {
19318        let state = review_state(&["ignored"]);
19319        let f = facts(
19320            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19321            "s",
19322        );
19323        let m = pr_message_with(&state, 'A', Some(&f));
19324        assert!(m.body.len() < 30_000, "{}", m.body.len());
19325        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19326        assert!(
19327            m.body.contains("- ... 1 more commit(s)"),
19328            "{}",
19329            m.body.len()
19330        );
19331        assert!(m.body.ends_with("magi:candidate-a\n"));
19332    }
19333
19334    #[test]
19335    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19336        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19337        assert_eq!(m.title, "feat: x");
19338        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19339        assert!(!m.body.contains("Diff stat"), "{}", m.body);
19340        assert!(!m.body.contains("no task statement"), "{}", m.body);
19341    }
19342
19343    #[test]
19344    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19345        let mut state = review_state(&["日本語の件名"]);
19346        state.candidates[0].branch = "fix/macos-login-path".to_owned();
19347        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19348        state.candidates[0].branch = "機能/ログイン".to_owned();
19349        assert!(
19350            pr_message(&state, 'A')
19351                .title
19352                .starts_with("chore: land candidate A")
19353        );
19354    }
19355
19356    #[test]
19357    fn pr_message_review_fixes_survive_a_clean_final_round() {
19358        let mut state = review_state(&["feat: x"]);
19359        state.reviews = vec![
19360            round_with_notes(1, Some("handled the PATH case")),
19361            round_with_notes(2, None),
19362        ];
19363        let body = pr_message(&state, 'A').body;
19364        assert!(
19365            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19366            "{body}"
19367        );
19368        assert!(body.contains("### Round 1"), "{body}");
19369
19370        state.reviews = vec![
19371            round_with_notes(1, Some("first fix")),
19372            round_with_notes(2, Some("")),
19373            round_with_notes(3, Some("second fix")),
19374            round_with_notes(4, None),
19375        ];
19376        let body = pr_message(&state, 'A').body;
19377        assert!(
19378            body.contains("### Round 1") && !body.contains("first fix"),
19379            "{body}"
19380        );
19381        assert!(
19382            body.contains("### Round 3") && !body.contains("second fix"),
19383            "{body}"
19384        );
19385        assert!(body.contains("### Round 2"), "{body}");
19386    }
19387
19388    const FINAL: &str = "abcdef1234567890";
19389
19390    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
19391        let mut r = round_with_notes(round, None);
19392        r.verified_head = verified.map(str::to_owned);
19393        r
19394    }
19395
19396    fn ok_gate(state: &mut RunState, head: Option<&str>) {
19397        state.gate = vec![CommandOutcome {
19398            command: "gate".to_owned(),
19399            code: Some(0),
19400            output_tail: String::new(),
19401            duration_ms: 0,
19402            resource_blocked: false,
19403        }];
19404        state.gate_ran = true;
19405        state.gate_head = head.map(str::to_owned);
19406    }
19407
19408    fn body_at(state: &RunState) -> String {
19409        pr_message_raw(
19410            state,
19411            'A',
19412            None,
19413            crate::pr_outcome::FinalHead::At(FINAL),
19414            ChangeNote::NotAsked,
19415        )
19416        .body
19417    }
19418
19419    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
19420
19421    #[test]
19422    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
19423        let mut state = review_state(&["feat: x"]);
19424        state.reviews = vec![
19425            round_with_notes(1, Some(SANDBOXED)),
19426            passing_round(2, Some(FINAL)),
19427        ];
19428        ok_gate(&mut state, Some(FINAL));
19429        let body = body_at(&state);
19430        assert!(!body.contains("Handled the PATH case."), "{body}");
19431        assert!(!body.contains("could not commit"), "{body}");
19432        assert!(!body.contains("could not run"), "{body}");
19433        assert!(
19434            body.contains("The fix is committed on the branch"),
19435            "{body}"
19436        );
19437        assert!(!body.contains("sandbox"), "{body}");
19438        assert!(
19439            body.contains("Tests (`verify.e2e`): passed on the final commit"),
19440            "{body}"
19441        );
19442        assert!(
19443            body.contains("Gate (`verify.gate`): passed on the final commit"),
19444            "{body}"
19445        );
19446        // The raw report is untouched in the record.
19447        assert!(
19448            state.reviews[0]
19449                .fix
19450                .as_ref()
19451                .unwrap()
19452                .notes
19453                .contains("could not commit")
19454        );
19455    }
19456
19457    #[test]
19458    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
19459        let mut state = review_state(&["feat: x"]);
19460        let mut r1 = round_with_notes(1, Some(SANDBOXED));
19461        r1.fix.as_mut().unwrap().committed = false;
19462        let mut r2 = passing_round(2, Some(FINAL));
19463        r2.e2e[0].code = Some(1);
19464        state.reviews = vec![r1, r2];
19465        let body = body_at(&state);
19466        assert!(!body.contains("could not commit"), "{body}");
19467        assert!(!body.contains("could not run"), "{body}");
19468        assert!(body.contains("This round produced no commit"), "{body}");
19469        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
19470        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
19471        assert!(!body.contains("passed"), "{body}");
19472    }
19473
19474    #[test]
19475    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
19476        let mut state = review_state(&["feat: x"]);
19477        state.reviews = vec![
19478            round_with_notes(1, Some(SANDBOXED)),
19479            passing_round(2, Some("0123456789abcdef")),
19480        ];
19481        ok_gate(&mut state, Some("0123456789abcdef"));
19482        let body = body_at(&state);
19483        assert!(!body.contains("HTTP tests could not run"), "{body}");
19484        // The commit claim is still settled by the structured record.
19485        assert!(!body.contains("could not commit"), "{body}");
19486        assert!(
19487            body.contains("passed on an earlier commit (`0123456`)"),
19488            "{body}"
19489        );
19490        assert!(!body.contains("passed on the final commit"), "{body}");
19491
19492        // A gate recorded before `gate_head` existed is "not recorded".
19493        ok_gate(&mut state, None);
19494        let body = body_at(&state);
19495        assert!(
19496            body.contains("passed on a commit that was not recorded"),
19497            "{body}"
19498        );
19499    }
19500
19501    #[test]
19502    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
19503        let state = state_with_summary(
19504            "add retries to the client",
19505            "TITLE: feat: retries\n\nDid it.",
19506        );
19507        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
19508        assert_eq!(m.title, "feat: retries");
19509        assert!(
19510            m.body.contains("<summary>Original task</summary>"),
19511            "{}",
19512            m.body
19513        );
19514        assert!(!m.body.contains("Commits under review"), "{}", m.body);
19515        assert!(
19516            m.body
19517                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
19518            "{}",
19519            m.body
19520        );
19521    }
19522
19523    #[test]
19524    fn pr_message_review_bounds_a_long_english_subject() {
19525        let long = format!("feat: {}", "word ".repeat(100));
19526        let m = pr_message(&review_state(&[&long]), 'A');
19527        assert!(m.title.starts_with("feat: word"), "{}", m.title);
19528        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19529    }
19530
19531    #[test]
19532    fn pr_message_implementation_run_is_unchanged_by_review_support() {
19533        let state = state_with_summary("add retries\n\ndetails", "- did some things");
19534        let m = pr_message(&state, 'A');
19535        assert_eq!(m.title, "add retries");
19536        assert!(m.body.contains("<summary>Original task</summary>"));
19537        assert!(!m.body.contains("Commits under review"));
19538        assert_eq!(landing_subject_source(&state), state.instruction);
19539    }
19540
19541    #[test]
19542    fn review_run_squash_subject_is_the_change_not_the_prompt() {
19543        let state = review_state(&["feat: the change", "fix: typo"]);
19544        let source = landing_subject_source(&state);
19545        assert_eq!(land::merge_subject("", &source), "feat: the change");
19546        assert_eq!(
19547            land::merge_subject("magi: candidate A (uncommitted work)", &source),
19548            "feat: the change"
19549        );
19550        // An operator's rename still wins.
19551        assert_eq!(
19552            land::merge_subject("feat: renamed by hand", &source),
19553            "feat: renamed by hand"
19554        );
19555        let blank = review_state(&["日本語"]);
19556        assert!(
19557            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
19558        );
19559    }
19560
19561    #[test]
19562    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
19563        let state = review_state(&["feat: the change"]);
19564        let source = landing_subject_source(&state);
19565        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
19566        assert_eq!(
19567            land::merge_subject(landing_title(&state, old), &source),
19568            "feat: the change"
19569        );
19570        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
19571        let task = state_with_summary("add retries", "");
19572        assert_eq!(landing_title(&task, old), old);
19573    }
19574
19575    #[test]
19576    fn pr_message_describes_the_change_not_the_task() {
19577        let state = state_with_summary(
19578            "今回やってほしいこと: results projector を直す",
19579            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
19580        );
19581        let m = pr_message(&state, 'A');
19582        assert_eq!(m.title, "fix(web): batch the runs list reads");
19583        assert!(
19584            m.body.starts_with("## Summary\n\n- reads run.json once"),
19585            "{}",
19586            m.body
19587        );
19588        assert!(!m.body.contains("TITLE:"), "{}", m.body);
19589        let task_at = m.body.find("今回やってほしいこと").unwrap();
19590        let details_at = m.body.find("<details>").unwrap();
19591        assert!(
19592            details_at < task_at,
19593            "the task lives inside <details>: {}",
19594            m.body
19595        );
19596        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
19597        assert!(m.body.contains("magi:candidate-a"));
19598    }
19599
19600    #[test]
19601    fn pr_message_falls_back_to_the_task_without_a_title_line() {
19602        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
19603        let m = pr_message(&state, 'A');
19604        assert_eq!(m.title, "add retries");
19605        assert!(
19606            m.body.contains("## Summary\n\n- did some things"),
19607            "{}",
19608            m.body
19609        );
19610
19611        let none = RunState::new(
19612            PathBuf::from("/repo"),
19613            "main".to_owned(),
19614            "abc1234".to_owned(),
19615            "add retries".to_owned(),
19616            Config::default(),
19617        );
19618        let m = pr_message(&none, 'A');
19619        assert_eq!(m.title, "add retries");
19620        assert!(!m.body.contains("## Summary"), "{}", m.body);
19621    }
19622
19623    #[test]
19624    fn pr_message_refuses_the_candidate_commit_subject() {
19625        for bad in [
19626            "TITLE: magi: candidate A (uncommitted work)",
19627            "TITLE: chore: stuff (uncommitted work)",
19628            "TITLE:   ",
19629        ] {
19630            let state = state_with_summary("add retries", bad);
19631            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
19632        }
19633    }
19634
19635    #[test]
19636    fn pr_message_bounds_a_very_long_task_and_title() {
19637        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
19638        let state = state_with_summary(&long, "- nothing");
19639        let m = pr_message(&state, 'A');
19640        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19641        assert!(!m.title.contains('\n'));
19642
19643        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
19644        let m = pr_message(&state, 'A');
19645        assert!(m.title.starts_with("feat: "));
19646        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19647        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
19648    }
19649
19650    fn long_title_of(instruction: &str) -> String {
19651        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
19652    }
19653
19654    #[test]
19655    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
19656        let first = "Make the landing path keep a readable title for long tasks";
19657        let line = format!(
19658            "{first}. {}",
19659            "And then keep going with more words ".repeat(20)
19660        );
19661        let t = long_title_of(&line);
19662        assert_eq!(t, first);
19663        assert!(!t.starts_with("chore: land"));
19664    }
19665
19666    #[test]
19667    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
19668        let line = "word ".repeat(200);
19669        let t = long_title_of(&line);
19670        assert!(t.ends_with("word..."), "{t}");
19671        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
19672    }
19673
19674    #[test]
19675    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
19676        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
19677            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
19678        }
19679    }
19680
19681    #[test]
19682    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
19683        let mut state = state_with_summary("再試行を追加する", "- nothing");
19684        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
19685        state.config.graph.github_language = "ja".into();
19686        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
19687    }
19688
19689    #[test]
19690    fn pr_message_title_limit_is_exact() {
19691        let at = "a".repeat(PR_TITLE_MAX);
19692        assert_eq!(long_title_of(&at), at);
19693        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
19694        assert!(over.ends_with("..."), "{over}");
19695        assert_eq!(over.chars().count(), PR_TITLE_MAX);
19696    }
19697
19698    #[test]
19699    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
19700        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
19701        let t = long_title_of(&line);
19702        assert!(t.ends_with("..."), "{t}");
19703        assert!(t.is_ascii(), "{t}");
19704    }
19705
19706    #[test]
19707    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
19708        let line = format!(
19709            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
19710            "plus more filler words ".repeat(20)
19711        );
19712        let t = long_title_of(&line);
19713        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
19714    }
19715
19716    #[test]
19717    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
19718        let line = format!("\n\n# {}", "title words ".repeat(40));
19719        let state = state_with_summary(&line, "- nothing");
19720        let m = pr_message(&state, 'A');
19721        assert!(m.title.starts_with("title words"), "{}", m.title);
19722        assert_eq!(
19723            land::merge_subject(&m.title, &landing_subject_source(&state)),
19724            m.title
19725        );
19726        // An operator's rename wins untouched.
19727        assert_eq!(
19728            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
19729            "feat: renamed by hand"
19730        );
19731    }
19732
19733    #[tokio::test]
19734    async fn github_text_rewrite_is_bounded_and_falls_back() {
19735        crate::run::pin_test_home();
19736        for (reply, accepted) in [
19737            (
19738                "TITLE: fix: retries\nAdd retries for failed requests.",
19739                true,
19740            ),
19741            (
19742                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
19743                false,
19744            ),
19745            (
19746                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
19747                true,
19748            ),
19749            (
19750                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
19751                true,
19752            ),
19753            (
19754                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
19755                true,
19756            ),
19757            (
19758                "TITLE: fix: retries\n日本語の説明です。 token=secret",
19759                false,
19760            ),
19761            ("TITLE: fix: retries\n", false),
19762        ] {
19763            let dir = tempfile::tempdir().unwrap();
19764            let mut runner = runner_at(RunStatus::Gating);
19765            runner.state = state_with_summary(
19766                "add retries",
19767                "TITLE: fix: retries\n日本語の説明を書きます。",
19768            );
19769            runner.state.repo = dir.path().to_owned();
19770            runner.state.candidates[0].worktree = dir.path().to_owned();
19771            let mut author = spec("alpha");
19772            author.command = vec![
19773                "sh".into(),
19774                "-c".into(),
19775                "printf '%s' \"$REWRITE_REPLY\"".into(),
19776            ];
19777            author.env.insert("REWRITE_REPLY".into(), reply.into());
19778            runner.state.config.agents = vec![author];
19779            seed_public_summary(&mut runner);
19780            let winner = runner.state.candidates[0].clone();
19781            let first = runner
19782                .guarded_pr_message(&winner, None, true)
19783                .await
19784                .unwrap();
19785            let message = if accepted {
19786                first.expect("a passing rewrite never asks")
19787            } else {
19788                // Withheld: the run parks on a question and posts nothing yet.
19789                assert!(first.is_none());
19790                assert!(runner.state.parked);
19791                let store = ask::Questions::open();
19792                let asked: Vec<_> = store
19793                    .list()
19794                    .into_iter()
19795                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19796                    .collect();
19797                assert_eq!(asked.len(), 1);
19798                assert!(!asked[0].detail.contains("secret"));
19799                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
19800                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
19801                assert_eq!(
19802                    asked[0]
19803                        .choices
19804                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
19805                    rewrite.is_some()
19806                );
19807                // Silence: the question lapses and the neutral text is used,
19808                // without a second rewrite call or a second question.
19809                store
19810                    .update(&asked[0].id, |q| {
19811                        q.abandon("test");
19812                        Ok(())
19813                    })
19814                    .unwrap();
19815                let again = runner
19816                    .guarded_pr_message(&winner, None, true)
19817                    .await
19818                    .unwrap()
19819                    .expect("an abandoned question falls back");
19820                let n = store
19821                    .list()
19822                    .into_iter()
19823                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19824                    .count();
19825                assert_eq!(n, 1, "asked once per rejected text");
19826                again
19827            };
19828            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
19829            assert!(!message.body.contains("secret"));
19830            assert!(!message.body.contains("prefix"));
19831            assert!(!message.title.contains("secret"));
19832            assert_eq!(
19833                message.body.contains("Add retries for failed requests."),
19834                accepted
19835            );
19836            assert_eq!(
19837                runner.state.seats["github-text-rewrite"].turns, 1,
19838                "only one rewrite invocation"
19839            );
19840            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
19841            if accepted {
19842                assert!(runner.state.github_text.is_none());
19843                assert!(!runner.state.parked);
19844                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
19845                assert!(message.body.contains("## Final verification"));
19846                assert!(message.body.contains("Evidence: final diff"));
19847                assert!(
19848                    message
19849                        .body
19850                        .contains(&format!("magi:run/{}", runner.state.id))
19851                );
19852            }
19853            if !accepted {
19854                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19855                assert!(
19856                    message
19857                        .body
19858                        .contains(&format!("magi:run/{}", runner.state.id))
19859                );
19860            }
19861        }
19862    }
19863
19864    fn scripted(id: &str, reply: &str) -> AgentSpec {
19865        let mut a = spec(id);
19866        a.command = vec![
19867            "sh".into(),
19868            "-c".into(),
19869            "printf '%s' \"$REWRITE_REPLY\"".into(),
19870        ];
19871        a.env.insert("REWRITE_REPLY".into(), reply.into());
19872        a
19873    }
19874
19875    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
19876    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
19877
19878    /// Gate tests inject generated public output, not internal reports.
19879    fn seed_public_summary(runner: &mut Runner) {
19880        let repo = &runner.state.repo;
19881        let g = |args: &[&str]| {
19882            let out = std::process::Command::new("git")
19883                .args(args)
19884                .current_dir(repo)
19885                .quiet()
19886                .output()
19887                .unwrap();
19888            assert!(out.status.success(), "git {args:?}");
19889            String::from_utf8_lossy(&out.stdout).trim().to_owned()
19890        };
19891        g(&["init", "-q", "-b", "main"]);
19892        g(&[
19893            "-c",
19894            "user.name=Test",
19895            "-c",
19896            "user.email=test@example.invalid",
19897            "commit",
19898            "-q",
19899            "--allow-empty",
19900            "-m",
19901            "feat: fixture",
19902        ]);
19903        let head = g(&["rev-parse", "HEAD"]);
19904        g(&["update-ref", "refs/remotes/origin/main", &head]);
19905        let evidence = crate::pr_outcome::summary_evidence(
19906            &runner.state,
19907            crate::pr_outcome::FinalHead::At(&head),
19908        );
19909        let generated = &runner.state.candidates[0].summary;
19910        runner.state.change_summary = Some(crate::run::ChangeSummary {
19911            base: head.clone(),
19912            head,
19913            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
19914            title: summary_title(generated).unwrap(),
19915            body: summary_without_title(generated),
19916        });
19917    }
19918
19919    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19920        let mut runner = runner_at(RunStatus::Gating);
19921        runner.state = state_with_summary("add retries", JA_SUMMARY);
19922        if review_only {
19923            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19924        }
19925        runner.state.repo = dir.to_owned();
19926        runner.state.candidates[0].worktree = dir.to_owned();
19927        runner.state.config.agents = agents;
19928        seed_public_summary(&mut runner);
19929        runner
19930    }
19931
19932    fn gate_questions(runner: &Runner) -> usize {
19933        ask::Questions::open()
19934            .list()
19935            .into_iter()
19936            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19937            .count()
19938    }
19939
19940    #[tokio::test]
19941    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19942        crate::run::pin_test_home();
19943        let dir = tempfile::tempdir().unwrap();
19944        let mut runner = rewrite_runner(
19945            dir.path(),
19946            true,
19947            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19948        );
19949        runner.state.config.roles.language_judge =
19950            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19951        let winner = runner.state.candidates[0].clone();
19952        let msg = runner
19953            .guarded_pr_message(&winner, None, true)
19954            .await
19955            .unwrap()
19956            .expect("an English rewrite posts without asking");
19957        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19958        assert!(msg.body.contains("<details>kept</details>"));
19959        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19960        assert_eq!(gate_questions(&runner), 0);
19961        assert!(runner.state.github_text.is_none());
19962        assert!(!runner.state.seats.contains_key("impl-A"));
19963        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19964    }
19965
19966    #[tokio::test]
19967    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19968        crate::run::pin_test_home();
19969        let dir = tempfile::tempdir().unwrap();
19970        let mut runner = rewrite_runner(
19971            dir.path(),
19972            false,
19973            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19974        );
19975        runner.state.config.roles.language_judge =
19976            Some(crate::config::AgentChoice::One("beta".into()));
19977        runner.state.seats.insert(
19978            "impl-A".into(),
19979            SeatState::new("impl-A", "alpha", runner.state.seed),
19980        );
19981        let winner = runner.state.candidates[0].clone();
19982        let msg = runner
19983            .guarded_pr_message(&winner, None, true)
19984            .await
19985            .unwrap()
19986            .expect("rewritten");
19987        assert!(msg.body.contains("<details>kept</details>"));
19988        assert_eq!(runner.state.seats["impl-A"].turns, 0);
19989        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19990        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19991    }
19992
19993    #[tokio::test]
19994    async fn rewrite_reuses_a_resumable_implementer_session() {
19995        crate::run::pin_test_home();
19996        let dir = tempfile::tempdir().unwrap();
19997        let mut runner = rewrite_runner(
19998            dir.path(),
19999            false,
20000            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20001        );
20002        runner.state.config.roles.language_judge =
20003            Some(crate::config::AgentChoice::One("beta".into()));
20004        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20005        seat.turns = 1;
20006        runner.state.seats.insert("impl-A".into(), seat);
20007        let winner = runner.state.candidates[0].clone();
20008        runner
20009            .guarded_pr_message(&winner, None, true)
20010            .await
20011            .unwrap()
20012            .expect("rewritten");
20013        assert_eq!(runner.state.seats["impl-A"].turns, 2);
20014        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20015    }
20016
20017    #[tokio::test]
20018    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20019        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
20020        // session, was out of quota, and nothing else was asked.
20021        crate::run::pin_test_home();
20022        let dir = tempfile::tempdir().unwrap();
20023        let mut exhausted = spec("alpha");
20024        exhausted.command = vec!["false".into()];
20025        let mut runner = rewrite_runner(
20026            dir.path(),
20027            false,
20028            vec![exhausted, scripted("beta", EN_REWRITE)],
20029        );
20030        // The implementer's own agent is in the role too: it is not asked twice.
20031        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20032            "alpha".into(),
20033            "beta".into(),
20034        ]));
20035        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20036        seat.turns = 1;
20037        runner.state.seats.insert("impl-A".into(), seat);
20038        let winner = runner.state.candidates[0].clone();
20039        let msg = runner
20040            .guarded_pr_message(&winner, None, true)
20041            .await
20042            .unwrap()
20043            .expect("the role rewrites after the implementer fails");
20044        assert!(msg.body.contains("`src/net.rs`"));
20045        assert_eq!(
20046            runner.state.seats["impl-A"].turns, 2,
20047            "the implementer was tried once"
20048        );
20049        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20050        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20051    }
20052
20053    #[tokio::test]
20054    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20055        crate::run::pin_test_home();
20056        let dir = tempfile::tempdir().unwrap();
20057        let mut broken = spec("bad");
20058        broken.command = vec!["false".into()];
20059        let mut runner = rewrite_runner(
20060            dir.path(),
20061            true,
20062            vec![
20063                broken,
20064                scripted("noformat", "Add retries."),
20065                scripted("good", EN_REWRITE),
20066            ],
20067        );
20068        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20069            "bad".into(),
20070            "noformat".into(),
20071            "good".into(),
20072        ]));
20073        let winner = runner.state.candidates[0].clone();
20074        let msg = runner
20075            .guarded_pr_message(&winner, None, true)
20076            .await
20077            .unwrap()
20078            .expect("the third agent rewrites");
20079        assert!(msg.body.contains("`src/net.rs`"));
20080        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20081        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20082    }
20083
20084    #[tokio::test]
20085    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20086        crate::run::pin_test_home();
20087        // (agents, role): still Japanese; no TITLE; nothing resolvable.
20088        for (agents, role, saved) in [
20089            (
20090                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20091                Some("beta"),
20092                true,
20093            ),
20094            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20095            (Vec::new(), None, false),
20096        ] {
20097            let dir = tempfile::tempdir().unwrap();
20098            let mut runner = rewrite_runner(dir.path(), true, agents);
20099            runner.state.config.roles.language_judge =
20100                role.map(|r| crate::config::AgentChoice::One(r.into()));
20101            let winner = runner.state.candidates[0].clone();
20102            let first = runner
20103                .guarded_pr_message(&winner, None, true)
20104                .await
20105                .unwrap();
20106            assert!(first.is_none(), "withheld text parks the run");
20107            assert!(runner.state.parked);
20108            assert_eq!(gate_questions(&runner), 1);
20109            let g = runner.state.github_text.as_ref().unwrap();
20110            assert_eq!(g.rewrite.is_some(), saved);
20111            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20112        }
20113    }
20114
20115    #[tokio::test]
20116    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20117        crate::run::pin_test_home();
20118        let dir = tempfile::tempdir().unwrap();
20119        let mut runner = runner_at(RunStatus::Gating);
20120        runner.state =
20121            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20122        runner.state.repo = dir.path().to_owned();
20123        runner.state.candidates[0].worktree = dir.path().to_owned();
20124        let mut author = spec("alpha");
20125        author.command = vec![
20126            "sh".into(),
20127            "-c".into(),
20128            "printf '%s' \"$REWRITE_REPLY\"".into(),
20129        ];
20130        author.env.insert(
20131            "REWRITE_REPLY".into(),
20132            format!(
20133                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20134                "English sentence. ".repeat(600)
20135            ),
20136        );
20137        runner.state.config.agents = vec![author];
20138        seed_public_summary(&mut runner);
20139        let winner = runner.state.candidates[0].clone();
20140        assert!(
20141            runner
20142                .guarded_pr_message(&winner, None, true)
20143                .await
20144                .unwrap()
20145                .is_none()
20146        );
20147        let record = runner.state.github_text.clone().unwrap();
20148        let rewrite = record.rewrite.clone().unwrap();
20149        assert_eq!(rewrite.failed, ["body-language"]);
20150        assert!(
20151            rewrite
20152                .body
20153                .ends_with(&format!("magi:run/{}", runner.state.id))
20154        );
20155        let store = ask::Questions::open();
20156        let q = store.get(record.question.as_ref().unwrap()).unwrap();
20157        assert!(
20158            q.choices
20159                .contains(&crate::github_text::USE_REWRITE.to_owned())
20160        );
20161        for label in [
20162            "ORIGINAL",
20163            "FALLBACK",
20164            "REWRITE",
20165            "Rules still failed: body-language",
20166            "truncated",
20167        ] {
20168            assert!(q.detail.contains(label), "{label}");
20169        }
20170        let artifact = std::fs::read_to_string(crate::run::artifact_path(
20171            &runner.state,
20172            &crate::github_text::artifact_name(&record.fingerprint),
20173        ))
20174        .unwrap();
20175        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20176        // Restart from serialized state, then retry after a bad owner title.
20177        runner.state =
20178            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20179        store
20180            .update(&q.id, |q| {
20181                q.say("まだ日本語")?;
20182                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20183            })
20184            .unwrap();
20185        assert!(
20186            runner
20187                .guarded_pr_message(&winner, None, true)
20188                .await
20189                .unwrap()
20190                .is_none()
20191        );
20192        let retry = runner.state.github_text.as_ref().unwrap();
20193        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20194        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20195        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20196        assert!(
20197            q.posting_comparison
20198                .as_ref()
20199                .unwrap()
20200                .matches_detail(&q.detail)
20201        );
20202        store
20203            .update(&q.id, |q| {
20204                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20205            })
20206            .unwrap();
20207        let message = runner
20208            .guarded_pr_message(&winner, None, true)
20209            .await
20210            .unwrap()
20211            .unwrap();
20212        assert_eq!(message.title, rewrite.title);
20213        assert_eq!(message.body, rewrite.body);
20214        assert_eq!(
20215            runner.state.github_text.as_ref().unwrap().choice,
20216            Some(GithubTextChoice::Rewrite)
20217        );
20218        let again = runner
20219            .guarded_pr_message(&winner, None, true)
20220            .await
20221            .unwrap()
20222            .unwrap();
20223        assert_eq!(again.title, message.title);
20224        assert_eq!(again.body, message.body);
20225        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20226        assert!(
20227            !runner
20228                .state
20229                .events
20230                .iter()
20231                .any(|e| e.message.contains(&rewrite.title)
20232                    || e.message.contains("English sentence."))
20233        );
20234    }
20235
20236    #[tokio::test]
20237    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20238        crate::run::pin_test_home();
20239        let dir = tempfile::tempdir().unwrap();
20240        let mut runner = runner_at(RunStatus::Gating);
20241        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20242        runner.state.repo = dir.path().to_owned();
20243        runner.state.candidates[0].worktree = dir.path().to_owned();
20244        seed_public_summary(&mut runner);
20245        let winner = runner.state.candidates[0].clone();
20246        assert!(
20247            runner
20248                .guarded_pr_message(&winner, None, true)
20249                .await
20250                .unwrap()
20251                .is_none()
20252        );
20253        let record = runner.state.github_text.as_ref().unwrap();
20254        assert!(record.rewrite.is_none());
20255        // Records written before the additive field deserialize with no rewrite.
20256        let mut old = serde_json::to_value(record).unwrap();
20257        old.as_object_mut().unwrap().remove("rewrite");
20258        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20259        assert!(old.rewrite.is_none());
20260        let id = record.question.clone().unwrap();
20261        ask::Questions::open()
20262            .update(&id, |q| {
20263                q.choices.push(crate::github_text::USE_REWRITE.into());
20264                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20265            })
20266            .unwrap();
20267        let message = runner
20268            .guarded_pr_message(&winner, None, true)
20269            .await
20270            .unwrap()
20271            .unwrap();
20272        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20273        assert_eq!(
20274            runner.state.github_text.as_ref().unwrap().choice,
20275            Some(GithubTextChoice::Fallback)
20276        );
20277    }
20278
20279    #[tokio::test]
20280    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20281        crate::run::pin_test_home();
20282        let dir = tempfile::tempdir().unwrap();
20283        let mut runner = runner_at(RunStatus::Gating);
20284        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20285        runner.state.repo = dir.path().to_owned();
20286        runner.state.candidates[0].worktree = dir.path().to_owned();
20287        seed_public_summary(&mut runner);
20288        let winner = runner.state.candidates[0].clone();
20289        assert!(
20290            runner
20291                .guarded_pr_message(&winner, None, true)
20292                .await
20293                .unwrap()
20294                .is_none()
20295        );
20296        let store = ask::Questions::open();
20297        let find = |run: &str| {
20298            store
20299                .list()
20300                .into_iter()
20301                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20302                .unwrap()
20303        };
20304        let run = runner.state.id.clone();
20305        let q = find(&run);
20306        let fp = runner
20307            .state
20308            .github_text
20309            .as_ref()
20310            .unwrap()
20311            .fingerprint
20312            .clone();
20313        let name = crate::github_text::artifact_name(&fp);
20314        let path = crate::run::artifact_path(&runner.state, &name);
20315        assert!(q.detail.contains(&path.display().to_string()));
20316        assert!(q.detail.contains("title-language: title"));
20317        let saved = std::fs::read_to_string(&path).expect("artifact written");
20318        assert!(saved.contains("日本語のタイトル"));
20319        assert!(q.cwd.is_none());
20320        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20321        // A bad replacement is refused and asked about once more.
20322        let say = |id: &str, text: &str, choice: &str| {
20323            store
20324                .update(id, |q| {
20325                    q.thread.push(ask::Turn {
20326                        who: ask::Who::Operator,
20327                        body: text.to_owned(),
20328                        at: jiff::Timestamp::now(),
20329                        note: None,
20330                    });
20331                    q.answer(ask::Answer::Choice(choice.to_owned()))
20332                })
20333                .unwrap();
20334        };
20335        say(&q.id, "まだ日本語", "use my text");
20336        assert!(
20337            runner
20338                .guarded_pr_message(&winner, None, true)
20339                .await
20340                .unwrap()
20341                .is_none()
20342        );
20343        let second = find(&run);
20344        assert_ne!(second.id, q.id);
20345        assert!(second.detail.contains("did not pass"));
20346        // The retry still compares the original text; the rejected
20347        // replacement is named by category only.
20348        assert!(second.detail.contains("日本語のタイトル"));
20349        assert!(!second.detail.contains("まだ日本語"));
20350        assert!(second.detail.contains(&path.display().to_string()));
20351        say(&second.id, "fix: retry failed requests", "use my text");
20352        let message = runner
20353            .guarded_pr_message(&winner, None, true)
20354            .await
20355            .unwrap()
20356            .expect("a vetted replacement is posted");
20357        assert_eq!(message.title, "fix: retry failed requests");
20358        assert_eq!(
20359            runner
20360                .state
20361                .github_text
20362                .as_ref()
20363                .unwrap()
20364                .chosen_title
20365                .as_deref(),
20366            Some("fix: retry failed requests")
20367        );
20368        assert!(runner.state.github_text.as_ref().unwrap().resolved);
20369        // Resumed again: the saved decision is applied, nothing is re-asked.
20370        let again = runner
20371            .guarded_pr_message(&winner, None, true)
20372            .await
20373            .unwrap()
20374            .unwrap();
20375        assert_eq!(again.title, message.title);
20376        assert_eq!(
20377            store
20378                .list()
20379                .into_iter()
20380                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
20381                .count(),
20382            2
20383        );
20384    }
20385
20386    #[test]
20387    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
20388        // What magi itself writes stays English under any configured language,
20389        // so a future localisation of these headings fails here. (The agents'
20390        // own text is also checked by the posting gate.)
20391        let mut state = state_with_summary(
20392            "add retries",
20393            "TITLE: fix(web): batch reads\n- reads run.json once",
20394        );
20395        state.config.graph.language = "ja".to_owned();
20396        let m = pr_message(&state, 'A');
20397        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
20398        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
20399
20400        // The task is the operator's own text: it goes in untouched, and the
20401        // fallback title (no summary) may be in its language too.
20402        let task = "今回やってほしいこと: results projector を直す";
20403        let mut state = state_with_summary(task, "- no title line");
20404        state.config.graph.language = "ja".to_owned();
20405        let m = pr_message(&state, 'A');
20406        assert_eq!(
20407            m.title,
20408            format!("chore: land candidate A of run {}", state.id)
20409        );
20410        assert!(
20411            m.body.contains(&format!(
20412                "<summary>Original task</summary>\n\n{task}\n\n</details>"
20413            )),
20414            "{}",
20415            m.body
20416        );
20417    }
20418
20419    #[test]
20420    fn pr_message_scrubs_home_paths_and_addresses() {
20421        let state = state_with_summary(
20422            "fix it in /Users/someone/src/x",
20423            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
20424        );
20425        let m = pr_message(&state, 'A');
20426        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
20427            assert!(!m.body.contains(leak), "{}", m.body);
20428        }
20429        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
20430    }
20431
20432    #[test]
20433    fn pr_message_survives_a_task_that_closes_details() {
20434        let state = state_with_summary("a </details> b", "TITLE: fix: x");
20435        let m = pr_message(&state, 'A');
20436        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
20437    }
20438
20439    #[test]
20440    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
20441        let cmd = manual_merge_command(
20442            MergeStyle::Squash,
20443            Path::new("/repo"),
20444            "b",
20445            "fix: \"quoted\" $(x) `y`\n\nbody",
20446        );
20447        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
20448    }
20449
20450    #[test]
20451    fn manual_merge_command_matches_the_configured_style() {
20452        let repo = Path::new("/repo");
20453        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
20454
20455        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
20456        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
20457
20458        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
20459        assert_eq!(
20460            squash,
20461            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
20462             \"Merge magi run 0832 (candidate A)\""
20463        );
20464
20465        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
20466        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
20467    }
20468
20469    #[test]
20470    fn a_nudge_gets_a_quarter_of_the_budget() {
20471        // The judge and implement budgets magi ships with.
20472        assert_eq!(retry_budget(secs(1200), true), secs(300));
20473        assert_eq!(retry_budget(secs(3600), true), secs(900));
20474    }
20475
20476    #[test]
20477    fn a_resent_prompt_keeps_the_whole_budget() {
20478        // The seat kept no context, so the retry is the original job again and
20479        // shortening it would only guarantee a second failure.
20480        assert_eq!(retry_budget(secs(1200), false), secs(1200));
20481        assert_eq!(retry_budget(secs(60), false), secs(60));
20482    }
20483
20484    #[test]
20485    fn the_floor_never_exceeds_the_original_budget() {
20486        // A short configured timeout must not be *raised* by the floor: the
20487        // operator asked for a bound, and a retry may not outlast the attempt
20488        // it is retrying.
20489        assert_eq!(retry_budget(secs(60), true), secs(60));
20490        assert_eq!(retry_budget(secs(480), true), secs(120));
20491        assert_eq!(retry_budget(secs(0), true), secs(0));
20492    }
20493
20494    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
20495        agent::CommandEvidence {
20496            id: "item1".to_owned(),
20497            description: "cargo test".to_owned(),
20498            exit_code,
20499            result_summary: String::new(),
20500            source: "codex".to_owned(),
20501        }
20502    }
20503
20504    #[test]
20505    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
20506        // No evidence is not the same fact as unconfirmed evidence: a
20507        // backend with no adapter, or a reply that ran no commands at all,
20508        // must not be misread as carrying a dangling job.
20509        assert!(!has_unconfirmed_command(&[]));
20510    }
20511
20512    #[test]
20513    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
20514        // Deliberately not a check on the exit code's *value*: a fixer
20515        // legitimately runs something that fails mid-iteration before it
20516        // succeeds, and that must never by itself reopen a valid report.
20517        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
20518        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
20519        assert!(!has_unconfirmed_command(&[
20520            evidence(Some(0)),
20521            evidence(Some(101))
20522        ]));
20523    }
20524
20525    #[test]
20526    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
20527        assert!(has_unconfirmed_command(&[
20528            evidence(Some(0)),
20529            evidence(None)
20530        ]));
20531    }
20532
20533    #[test]
20534    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
20535        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20536        assert_eq!(
20537            verified_noop_claim(true, &[], text).as_deref(),
20538            Some("already fixed by b32cfc4, on main.")
20539        );
20540    }
20541
20542    #[test]
20543    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
20544        // A timeout or a bad exit code reads as the ordinary loss it is,
20545        // whatever the reply's own prose claims.
20546        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20547        assert!(verified_noop_claim(false, &[], text).is_none());
20548    }
20549
20550    #[test]
20551    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
20552        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20553        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
20554        // A confirmed command alongside the marker is fine.
20555        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
20556    }
20557
20558    #[test]
20559    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
20560        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
20561    }
20562
20563    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
20564    /// pair, in order, labelled A, B, C, ...
20565    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
20566        runner.state.candidates = shape
20567            .iter()
20568            .enumerate()
20569            .map(|(i, &(empty, verified))| Candidate {
20570                index: i,
20571                label: (b'A' + i as u8) as char,
20572                agent: "sonnet".to_owned(),
20573                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
20574                worktree: PathBuf::from(format!("/wt/{i}")),
20575                summary: String::new(),
20576                stat: String::new(),
20577                files: 0,
20578                commits: 0,
20579                empty,
20580                failed: None,
20581                verified_noop: verified.map(str::to_owned),
20582                duration_ms: 0,
20583                folded: false,
20584            })
20585            .collect();
20586    }
20587
20588    #[test]
20589    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
20590        ask_test_home();
20591        let mut runner = runner_at(RunStatus::Implementing);
20592        set_candidates(
20593            &mut runner,
20594            &[
20595                (true, Some("already on main at b32cfc4")),
20596                (true, Some("same fix, see the existing test")),
20597            ],
20598        );
20599
20600        runner
20601            .after_implement()
20602            .expect("a verified no-op is not an error");
20603
20604        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
20605    }
20606
20607    #[test]
20608    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
20609        ask_test_home();
20610        let mut runner = runner_at(RunStatus::Implementing);
20611        // Candidate A declares a verified no-op; candidate B simply wrote
20612        // nothing and said nothing about why. One candidate's claim is not
20613        // the whole run's agreement.
20614        set_candidates(
20615            &mut runner,
20616            &[(true, Some("already on main at b32cfc4")), (true, None)],
20617        );
20618
20619        let err = runner
20620            .after_implement()
20621            .expect_err("an unverified empty candidate must still fail the run");
20622
20623        assert!(
20624            err.to_string().contains("no candidate produced a change"),
20625            "{err}"
20626        );
20627        assert_eq!(runner.state.status, RunStatus::Failed);
20628    }
20629
20630    #[test]
20631    fn after_implement_still_fails_an_ordinary_all_empty_run() {
20632        ask_test_home();
20633        let mut runner = runner_at(RunStatus::Implementing);
20634        set_candidates(&mut runner, &[(true, None), (true, None)]);
20635
20636        let err = runner
20637            .after_implement()
20638            .expect_err("no candidate declared anything; this is an ordinary failure");
20639
20640        assert!(
20641            err.to_string().contains("no candidate produced a change"),
20642            "{err}"
20643        );
20644        assert_eq!(runner.state.status, RunStatus::Failed);
20645    }
20646}
20647
20648/// A graft rollback whose restore could not be confirmed (see
20649/// [`Runner::graft`]).
20650#[derive(Debug)]
20651struct GraftRestoreFailed(String);
20652
20653impl std::fmt::Display for GraftRestoreFailed {
20654    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
20655        write!(
20656            f,
20657            "could not restore the winner to its pre-graft commit: {}",
20658            self.0
20659        )
20660    }
20661}
20662
20663impl std::error::Error for GraftRestoreFailed {}
20664
20665/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
20666/// not the agent: they never draw on an agent's own timeout, and running out of
20667/// them never reads as an agent failure.
20668#[cfg(test)]
20669mod resource_wait_tests {
20670    use super::*;
20671    use std::collections::BTreeMap;
20672
20673    fn sh(id: &str, script: &str) -> AgentSpec {
20674        AgentSpec {
20675            id: id.to_owned(),
20676            kind: crate::config::AgentKind::Command,
20677            model: None,
20678            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
20679            extra_args: Vec::new(),
20680            env: BTreeMap::new(),
20681            prompt_delivery: None,
20682            max_turn_secs: None,
20683        }
20684    }
20685
20686    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
20687        SeatJob {
20688            seat: SeatState::new("impl-A", &spec.id, 1),
20689            spec,
20690            cwd: dir.to_path_buf(),
20691            prompt: "do it".to_owned(),
20692            timeout,
20693            allow_write: true,
20694            sessions: false,
20695            artifacts: dir.join("art"),
20696            stem: "impl-A".to_owned(),
20697            handover: None,
20698        }
20699    }
20700
20701    fn state(resource_wait_secs: u64) -> RunState {
20702        let mut config = Config::default();
20703        config.graph.timeout_resource_wait = resource_wait_secs;
20704        RunState::new(
20705            PathBuf::from("/repo"),
20706            "main".to_owned(),
20707            "abc1234".to_owned(),
20708            "task".to_owned(),
20709            config,
20710        )
20711    }
20712
20713    fn hold(cache: &Path) -> crate::cache::Guard {
20714        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
20715        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
20716            crate::cache::AcquireOutcome::Acquired(g) => g,
20717            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
20718        }
20719    }
20720
20721    async fn run(
20722        jobs: Vec<SeatJob>,
20723        sem: Arc<Semaphore>,
20724        cache: Option<&Path>,
20725        st: &mut RunState,
20726    ) -> Vec<(usize, SeatState, AgentOutcome)> {
20727        let prompts = Prompts::default();
20728        let ctx = WaveCtx {
20729            run: "r",
20730            node: "implement",
20731            prompts: &prompts,
20732            cache,
20733            round: None,
20734            carry_seats: false,
20735            pauses: None,
20736        };
20737        wave(jobs, sem, &ctx, st, 0).await
20738    }
20739
20740    #[tokio::test]
20741    async fn verification_cache_wait_preserves_the_command_budget() {
20742        crate::run::pin_test_home();
20743        let tmp = tempfile::tempdir().unwrap();
20744        let cache = tmp.path().join("target");
20745        let guard = hold(&cache);
20746        let release = tokio::spawn(async move {
20747            tokio::time::sleep(Duration::from_millis(100)).await;
20748            guard.release();
20749        });
20750        let mut st = state(10);
20751        let budget = Duration::from_millis(50);
20752        let wait = repository_wait(&st, None);
20753        let (outcomes, _) = with_cache_lease(
20754            &mut st,
20755            Some(&cache),
20756            "gate",
20757            "gate",
20758            tmp.path(),
20759            "abc1234",
20760            budget,
20761            "test gate",
20762            &wait,
20763            |_, available| async move {
20764                assert_eq!(
20765                    available, budget,
20766                    "contention must not shorten verification"
20767                );
20768                (Vec::new(), false, Vec::new())
20769            },
20770        )
20771        .await
20772        .unwrap();
20773        release.await.unwrap();
20774        assert!(outcomes.is_empty(), "{outcomes:?}");
20775        assert!(
20776            st.events
20777                .iter()
20778                .any(|event| event.message.contains("waiting for"))
20779        );
20780        assert!(st.resource_wait.is_none());
20781        // The verification body no longer owns the lease after returning.
20782        hold(&cache).release();
20783    }
20784
20785    #[tokio::test]
20786    async fn verification_cache_wait_expires_without_starting_commands() {
20787        crate::run::pin_test_home();
20788        let tmp = tempfile::tempdir().unwrap();
20789        let cache = tmp.path().join("target");
20790        let _held = hold(&cache);
20791        let mut st = state(0);
20792        let wait = repository_wait(&st, None);
20793        let (outcomes, _) = with_cache_lease(
20794            &mut st,
20795            Some(&cache),
20796            "gate",
20797            "gate",
20798            tmp.path(),
20799            "abc1234",
20800            Duration::from_secs(60),
20801            "test gate",
20802            &wait,
20803            |_, _| async { panic!("commands must not start without the cache lease") },
20804        )
20805        .await
20806        .unwrap();
20807        assert_eq!(outcomes.len(), 1);
20808        assert!(outcomes[0].resource_blocked);
20809        assert!(st.resource_wait.is_none());
20810        assert!(matches!(
20811            st.resource_block.unwrap().cause,
20812            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
20813        ));
20814    }
20815
20816    #[tokio::test]
20817    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
20818        crate::run::pin_test_home();
20819        let tmp = tempfile::tempdir().unwrap();
20820        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
20821        let cache = tmp.path().join("cache");
20822        let mut state = state(21600);
20823        let wait = repository_wait(&state, None);
20824        let (outcomes, _) = with_cache_lease(
20825            &mut state,
20826            Some(&cache),
20827            "gate",
20828            "gate",
20829            tmp.path(),
20830            "head",
20831            Duration::from_secs(60),
20832            "gate",
20833            &wait,
20834            |_, _| async { panic!("freshness failure must not start verification") },
20835        )
20836        .await
20837        .unwrap();
20838        assert_eq!(outcomes.len(), 1);
20839        assert!(outcomes[0].resource_blocked);
20840        let block = state.resource_block.unwrap();
20841        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
20842        assert!(!block.description().contains("timed out"));
20843    }
20844
20845    #[tokio::test]
20846    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
20847        crate::run::pin_test_home();
20848        let tmp = tempfile::tempdir().unwrap();
20849        let cache = tmp.path().join("cache");
20850        let _held = hold(&cache);
20851        let mut state = state(21600);
20852        let pause = Pause::new();
20853        let interrupt = Pause::new();
20854        let cause = ParkCause::Operator;
20855        pause.park_for(cause.clone());
20856        let prompts = Prompts::default();
20857        let ctx = WaveCtx {
20858            run: "test",
20859            node: "fix",
20860            prompts: &prompts,
20861            cache: Some(&cache),
20862            round: None,
20863            carry_seats: false,
20864            pauses: Some((&pause, &interrupt)),
20865        };
20866        let result = wave(
20867            vec![job(
20868                sh("writer", "exit 99"),
20869                tmp.path(),
20870                Duration::from_secs(1),
20871            )],
20872            Arc::new(Semaphore::new(1)),
20873            &ctx,
20874            &mut state,
20875            0,
20876        )
20877        .await;
20878        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
20879        assert_eq!(result[0].1.turns, 0);
20880        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20881        assert_eq!(
20882            state.resource_block.unwrap().cause,
20883            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
20884        );
20885        assert!(state.active.is_empty());
20886    }
20887
20888    #[tokio::test]
20889    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
20890        crate::run::pin_test_home();
20891        let tmp = tempfile::tempdir().unwrap();
20892        let cache = tmp.path().join("target-a");
20893        let guard = hold(&cache);
20894        // Held for longer than the agent's whole 1s budget.
20895        tokio::spawn(async move {
20896            tokio::time::sleep(Duration::from_millis(1500)).await;
20897            guard.release();
20898        });
20899        let mut st = state(60);
20900        let results = run(
20901            vec![job(
20902                sh("alpha", "echo ok"),
20903                tmp.path(),
20904                Duration::from_secs(1),
20905            )],
20906            Arc::new(Semaphore::new(1)),
20907            Some(&cache),
20908            &mut st,
20909        )
20910        .await;
20911        assert!(
20912            matches!(results[0].2, AgentOutcome::Ok(_)),
20913            "the wait must not be charged to the agent"
20914        );
20915        assert!(st.resource_block.is_none());
20916        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20917        assert!(st.resource_wait.is_none() && st.active.is_empty());
20918    }
20919
20920    #[tokio::test]
20921    #[cfg(unix)]
20922    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20923        use std::os::fd::AsRawFd as _;
20924        crate::run::pin_test_home();
20925        let tmp = tempfile::tempdir().unwrap();
20926        crate::git::git(tmp.path(), &["init"]).await.unwrap();
20927        let common = tmp.path().join(".git");
20928        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20929        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20930        assert_eq!(
20931            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20932            0
20933        );
20934        let mut st = state(1);
20935        st.config.coordination = crate::config::test_coordination();
20936        let mut reader = job(
20937            sh("reader", "echo must-not-run"),
20938            tmp.path(),
20939            Duration::from_secs(30),
20940        );
20941        reader.allow_write = false;
20942        let results = run(
20943            vec![reader.clone()],
20944            Arc::new(Semaphore::new(1)),
20945            None,
20946            &mut st,
20947        )
20948        .await;
20949        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20950        assert_eq!(results[0].1.turns, 0);
20951        assert!(FailClass::of(&results[0].2).is_none());
20952        assert_eq!(
20953            st.resource_block.as_ref().unwrap().kind,
20954            "repository coordination"
20955        );
20956        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20957        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20958        drop(external);
20959        let editor = crate::repo_lock::SeatLease::for_checkout(
20960            tmp.path(),
20961            true,
20962            crate::config::test_coordination().as_ref(),
20963        )
20964        .await
20965        .unwrap();
20966        let pause = Pause::new();
20967        let interrupt = Pause::new();
20968        let prompts = Prompts::default();
20969        let ctx = WaveCtx {
20970            run: "r",
20971            node: "review",
20972            prompts: &prompts,
20973            cache: None,
20974            round: None,
20975            carry_seats: false,
20976            pauses: Some((&pause, &interrupt)),
20977        };
20978        let (results, ()) = tokio::join!(
20979            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20980            async {
20981                tokio::time::sleep(Duration::from_millis(50)).await;
20982                pause.park();
20983            }
20984        );
20985        assert!(
20986            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20987        );
20988        assert_eq!(results[0].1.turns, 0);
20989        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20990        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20991        drop(editor);
20992    }
20993
20994    #[tokio::test]
20995    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20996        crate::run::pin_test_home();
20997        let tmp = tempfile::tempdir().unwrap();
20998        let cache = tmp.path().join("target-b");
20999        let _held = hold(&cache);
21000        let mut st = state(1);
21001        let results = run(
21002            vec![job(
21003                sh("alpha", "echo ok"),
21004                tmp.path(),
21005                Duration::from_secs(1),
21006            )],
21007            Arc::new(Semaphore::new(1)),
21008            Some(&cache),
21009            &mut st,
21010        )
21011        .await;
21012        let (_, seat, out) = &results[0];
21013        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21014        assert!(wave_blocked(&results));
21015        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21016        assert_eq!(seat.turns, 0, "nothing was asked");
21017        assert!(
21018            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21019            "no agent was spawned"
21020        );
21021        assert!(FailClass::of(out).is_none(), "not a failure class");
21022        let block = st.resource_block.as_ref().expect("recorded for the resume");
21023        assert_eq!(block.kind, "shared build cache");
21024        assert!(matches!(
21025            block.cause,
21026            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21027        ));
21028        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21029        assert!(st.active.is_empty());
21030    }
21031
21032    #[tokio::test]
21033    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21034        crate::run::pin_test_home();
21035        let tmp = tempfile::tempdir().unwrap();
21036        let mut st = state(1);
21037        let results = run(
21038            vec![job(
21039                sh("alpha", "echo ok"),
21040                tmp.path(),
21041                Duration::from_secs(1),
21042            )],
21043            Arc::new(Semaphore::new(0)),
21044            None,
21045            &mut st,
21046        )
21047        .await;
21048        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21049        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21050        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21051    }
21052
21053    #[tokio::test]
21054    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21055        crate::run::pin_test_home();
21056        let tmp = tempfile::tempdir().unwrap();
21057        let mut st = state(60);
21058        let results = run(
21059            vec![job(
21060                sh("alpha", "sleep 30"),
21061                tmp.path(),
21062                Duration::from_secs(1),
21063            )],
21064            Arc::new(Semaphore::new(1)),
21065            None,
21066            &mut st,
21067        )
21068        .await;
21069        let out = &results[0].2;
21070        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21071        let class = FailClass::of(out).expect("a failure");
21072        assert_eq!(class, FailClass::Timeout);
21073        assert!(should_hand_over(None, &class), "existing handover policy");
21074        assert!(st.resource_block.is_none());
21075    }
21076}