Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615impl Runner {
616    /// Start a fresh run against `repo`.
617    pub async fn start(
618        repo: &Path,
619        instruction: String,
620        config: Config,
621        origin: Origin,
622    ) -> Result<Self> {
623        Self::start_naming(repo, instruction, "", config, origin).await
624    }
625
626    /// [`Runner::start`] for a queued task. Titles and instructions never
627    /// select implementation sources. The title argument is retained for callers.
628    pub async fn start_naming(
629        repo: &Path,
630        instruction: String,
631        _also_scan: &str,
632        config: Config,
633        origin: Origin,
634    ) -> Result<Self> {
635        let repo = git::toplevel(repo).await?;
636        let missing = agent::missing_programs(&config.agents);
637        if !missing.is_empty() {
638            bail!(
639                "these agent programs are not on PATH: {}. Fix the roster in \
640                 magi.toml or install them.",
641                missing.join(", ")
642            );
643        }
644        let base_branch =
645            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646                .await?;
647        let fetched_base = resolve_base(
648            &repo,
649            &base_branch,
650            &config.merge.remote,
651            config.coordination.as_ref(),
652        )
653        .await?;
654        let base_commit = fetched_base.commit.clone();
655        let roles = config.resolve_roles()?;
656        let max_parallel = config.graph.max_parallel.max(1);
657        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658        // Recorded before the first save, so a crash right after minting
659        // cannot leave a run with no origin. Legibility only: nothing reads it
660        // to decide anything.
661        state.fetched_base = Some(fetched_base);
662        state.origin = Some(origin);
663        state.reserve()?;
664        state.event("start", format!("run {} created", state.id));
665        state.save()?;
666        Ok(Self {
667            state,
668            roles,
669            sem: Arc::new(Semaphore::new(max_parallel)),
670            pause: Pause::new(),
671            interrupt: Pause::new(),
672        })
673    }
674
675    /// Open a review-only run against work that already exists on `branch`.
676    ///
677    /// The expensive half of the graph is the implement wave — measured at
678    /// 111 and 134 internal tool-loop turns on this repository, against a
679    /// handful for a judge or a reviewer. The cheap half is worth running on
680    /// hand-written work too, and there was no way to reach it.
681    ///
682    /// No new state and no schema change are needed: a run with **one** viable
683    /// candidate and a tally already decided degrades `execute` to exactly
684    /// review → gate → merge, because `judge` skips a single-candidate field,
685    /// `deliberate` has fewer than two first choices to reconcile, `vote`
686    /// returns early, `tally` is already present and `fold_losers` has no
687    /// losers. Resuming such a run therefore does the right thing as well.
688    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689        Self::review_taking_over(repo, branch, config, None, origin).await
690    }
691
692    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
693    /// at the same task still has `branch` checked out, its worktree is
694    /// released first if that is safe (see [`crate::handover`]), and the
695    /// review refuses with the reason if it is not. `None` is a hand-run
696    /// review: it has no earlier attempts, so only a worktree of a dead run
697    /// magi recorded itself can be released.
698    pub async fn review_taking_over(
699        repo: &Path,
700        branch: &str,
701        config: Config,
702        takeover: Option<crate::handover::Takeover>,
703        origin: Origin,
704    ) -> Result<Self> {
705        let repo = git::toplevel(repo).await?;
706        let missing = agent::missing_programs(&config.agents);
707        if !missing.is_empty() {
708            bail!(
709                "these agent programs are not on PATH: {}. Fix the roster in \
710                 magi.toml or install them.",
711                missing.join(", ")
712            );
713        }
714        let base_branch =
715            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716                .await?;
717        if base_branch == branch {
718            bail!("`{branch}` is the base branch; there is nothing to review against");
719        }
720        let fetched_base = resolve_base(
721            &repo,
722            &base_branch,
723            &config.merge.remote,
724            config.coordination.as_ref(),
725        )
726        .await?;
727        let base_commit = fetched_base.commit.clone();
728
729        let roles = config.resolve_roles()?;
730        let max_parallel = config.graph.max_parallel.max(1);
731        let mut state = RunState::new(
732            repo.clone(),
733            base_branch,
734            base_commit.clone(),
735            String::new(),
736            config,
737        );
738        state.fetched_base = Some(fetched_base);
739        state.origin = Some(origin);
740        state.reserve()?;
741
742        // Released before anything else touches the branch: a stale local
743        // branch is moved with `git branch -f`, which git refuses while an
744        // earlier attempt's worktree still has it checked out. Everything
745        // after this point that can fail puts the old run back.
746        // A hand-run review has no task, hence no earlier attempts, but a
747        // worktree of a dead run magi made may still be released.
748        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749            earlier: Vec::new(),
750            home: crate::run::home(),
751            choice: None,
752            task_instruction: None,
753        });
754        let wait = repository_wait(&state, None);
755        let coordination_config = state.config.coordination.clone();
756        let coordination =
757            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758                .await?;
759        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760        if let Some(released) = &released {
761            state.event(
762                "release",
763                format!(
764                    "took `{branch}` over from run {}: its worktree was released: {}",
765                    crate::run::short_of(&released.old_id),
766                    released.audit
767                ),
768            );
769        }
770        // The owner's answer to an earlier divergence question is applied
771        // here: after the release (git will not move a checked-out branch)
772        // and before the sync that would otherwise ask again.
773        if let Some(choice) = takeover.choice.as_ref()
774            && let Err(e) =
775                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
776                    .await
777        {
778            if let Some(released) = &released {
779                released.restore(&repo, branch).await;
780            }
781            return Err(e.context("applying the owner's answer about the diverged branch"));
782        }
783        // An earlier attempt of this task that published the task head, and
784        // whose work is the branch under review, hands that head over: the
785        // review run updates the same pull request. Without one (a legacy
786        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
787        // own head and its own open pull request is adopted as before.
788        for id in takeover.earlier.iter().rev() {
789            if let Ok(old) = RunState::load_under(id, &takeover.home)
790                && let Some(head) = old.pr_head.clone()
791                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
792            {
793                state.pr_head = Some(head);
794                break;
795            }
796        }
797        drop(coordination);
798        let opened = Self::open_review(
799            &repo,
800            branch,
801            state,
802            roles,
803            max_parallel,
804            base_commit,
805            &takeover,
806        )
807        .await;
808        if opened.is_err()
809            && let Some(released) = &released
810        {
811            let _coordination = crate::repo_lock::SeatLease::operation_with(
812                &repo,
813                coordination_config.as_ref(),
814                &wait,
815            )
816            .await?;
817            released.restore(&repo, branch).await;
818        }
819        opened
820    }
821
822    /// Recover exactly the operator-selected public implementation, without touching old sources.
823    pub async fn recover_pr(
824        repo: &Path,
825        task: &str,
826        selection: crate::pr_recovery::Selection,
827        mut config: Config,
828    ) -> Result<Self> {
829        selection.configure(&mut config)?;
830        selection.check_task(task, &selection.instruction)?;
831        let base_branch =
832            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
833                .await?;
834        let fetched_base = resolve_base(
835            repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842        selection
843            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
844            .await?;
845        Box::pin(selection.verify_source(repo)).await?;
846        if let Some(proof) = &selection.source_proof {
847            anyhow::ensure!(
848                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
849                "source rebase base is outside the fetched landing base"
850            );
851        }
852        let roles = config.resolve_roles()?;
853        let max_parallel = config.graph.max_parallel.max(1);
854        let mut state = RunState::new(
855            repo.to_path_buf(),
856            base_branch,
857            base_commit.clone(),
858            selection.instruction.clone(),
859            config,
860        );
861        state.fetched_base = Some(fetched_base);
862        state.origin = Some(crate::run::Origin::queue(task));
863        state.pr_head = Some(selection.branch.clone());
864        state.pr_recovery = Some(selection.clone());
865        state.reserve()?;
866        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
867            // Never leave a reserved record behind as a phantom owner of the head.
868            state.discard_reserved();
869            return Err(e);
870        }
871        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
872    }
873
874    async fn open_selected(
875        repo: &Path,
876        state: RunState,
877        roles: ResolvedRoles,
878        max_parallel: usize,
879        base_commit: String,
880        selection: crate::pr_recovery::Selection,
881    ) -> Result<Self> {
882        let mut state = state;
883        let branch = state.branch_for('A');
884        let wait = repository_wait(&state, None);
885        let prepared = async {
886            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
887            git::git(repo, &["branch", &branch, selection.source()]).await?;
888            drop(coordination);
889            anyhow::Ok(())
890        }
891        .await;
892        if let Err(e) = prepared {
893            // Nothing was exposed yet: do not leave a reserved record as a
894            // phantom owner of the public head.
895            state.discard_reserved();
896            return Err(e);
897        }
898        state.event(
899            "recovery",
900            format!(
901                "selected implementation {} for {} (public lease {})",
902                selection.source(),
903                selection.url,
904                selection.head
905            ),
906        );
907        let takeover = crate::handover::Takeover {
908            earlier: Vec::new(),
909            home: crate::run::home(),
910            choice: None,
911            task_instruction: Some(selection.instruction.clone()),
912        };
913        if let Err(e) = state.save() {
914            state.discard_reserved();
915            return Err(e);
916        }
917        let id = state.id.clone();
918        let opened = Self::open_review(
919            repo,
920            &branch,
921            state,
922            roles,
923            max_parallel,
924            base_commit,
925            &takeover,
926        )
927        .await;
928        match opened {
929            Ok(runner) => Ok(runner),
930            Err(e) => {
931                if let Ok(mut partial) = RunState::load(&id) {
932                    partial.status = RunStatus::Blocked;
933                    partial.driver_exited = true;
934                    partial.event(
935                        "recovery",
936                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
937                    );
938                    partial.save()?;
939                }
940                Err(crate::pr_recovery::StartupFailed {
941                    run: id,
942                    cause: format!("{e:#}"),
943                }
944                .into())
945            }
946        }
947    }
948
949    /// What the newest earlier attempt that reviewed `branch` leaves for this
950    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
951    /// no earlier attempts. Reads the earlier record only: ownership of its
952    /// worktree and branch is [`crate::handover`]'s business.
953    async fn inherit(
954        repo: &Path,
955        branch: &str,
956        base_commit: &str,
957        takeover: &crate::handover::Takeover,
958    ) -> Option<crate::recovery::RecoveryRecord> {
959        use crate::recovery::{Relation, assess};
960        let newest = takeover.earlier.last()?;
961        let mut chosen = None;
962        for id in takeover.earlier.iter().rev() {
963            if let Ok(old) = RunState::load_under(id, &takeover.home)
964                && !old.reviews.is_empty()
965                && (old.winner().is_some_and(|w| w.branch == branch)
966                    || old.pr_head.as_deref() == Some(branch))
967            {
968                chosen = Some(old);
969                break;
970            }
971        }
972        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
973            .await
974            .unwrap_or_default();
975        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
976            Some(last) if last.head == tip => Relation::Same,
977            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
978            _ => Relation::Diverged,
979        };
980        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
981        Some(assess(from, chosen.as_ref(), base_commit, relation))
982    }
983
984    /// The half of [`Runner::review_taking_over`] that can fail after an
985    /// earlier attempt's worktree was released.
986    async fn open_review(
987        repo: &Path,
988        branch: &str,
989        mut state: RunState,
990        roles: ResolvedRoles,
991        max_parallel: usize,
992        base_commit: String,
993        takeover: &crate::handover::Takeover,
994    ) -> Result<Self> {
995        let wait = repository_wait(&state, None);
996        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
997        if state.pr_recovery.is_none() {
998            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
999        }
1000        // The commit subjects are the closest thing to a task statement that
1001        // existing work carries, and the reviewers are told as much.
1002        let start = review_base(
1003            repo,
1004            &state.config.merge.remote,
1005            &state.base_branch,
1006            &base_commit,
1007            branch,
1008        )
1009        .await;
1010        let log = git::log_oneline(repo, &start, branch)
1011            .await
1012            .unwrap_or_default();
1013        let log_text = if log.trim().is_empty() {
1014            "(no commit messages)"
1015        } else {
1016            log.trim()
1017        };
1018        let mut instruction = format!(
1019            "Review the work already on branch `{branch}`. There is no task \
1020             statement: what the change claims to do is whatever its commits \
1021             say.\n\n{log_text}"
1022        );
1023        // A retry of a task whose earlier attempt left reviews behind carries
1024        // that attempt's evidence, and the task's own statement with it: the
1025        // completion criteria are the task's, not whatever the commits claim.
1026        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1027            if let Some(task) = takeover
1028                .task_instruction
1029                .as_deref()
1030                .filter(|t| !t.trim().is_empty())
1031                && rec.reviewed_head.is_some()
1032            {
1033                instruction = format!(
1034                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1035                    task.trim()
1036                );
1037                rec.criteria_inherited = true;
1038            }
1039            state.event("recovery", rec.summary());
1040            state.recovery = Some(rec);
1041        }
1042        if let Some(selected) = &state.pr_recovery {
1043            instruction = selected.instruction.clone();
1044        }
1045        state.task_instruction = takeover.task_instruction.clone();
1046        state.instruction = instruction;
1047        state.reviewed_commits = Some(
1048            git::subjects(repo, &start, branch)
1049                .await
1050                .unwrap_or_default(),
1051        );
1052
1053        // A takeover must not forget a hold that a previous review observed,
1054        // even if repairs have since removed it from the branch.
1055        for id in &takeover.earlier {
1056            if let Ok(old) = RunState::load_under(id, &takeover.home)
1057                && old.repo == state.repo
1058                && (old.pr_head.as_deref() == Some(branch)
1059                    || old.winner().is_some_and(|w| w.branch == branch))
1060            {
1061                for hold in old.branch_owner_holds {
1062                    if !state.branch_owner_holds.contains(&hold) {
1063                        state.event(
1064                            "approval-policy",
1065                            format!(
1066                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1067                                hold.control.key(),
1068                                hold.file,
1069                                hold.head
1070                            ),
1071                        );
1072                        state.branch_owner_holds.push(hold);
1073                    }
1074                }
1075            }
1076        }
1077        let reviewed_head = git::rev_parse(repo, branch).await?;
1078        state.observe_branch_owner_holds(&reviewed_head).await;
1079
1080        // An attached worktree, so the fixer's commits land on the branch under
1081        // review rather than on a detached head nobody will look at again.
1082        let worktree = state.worktree_root().join("under-review");
1083        if let Some(parent) = worktree.parent() {
1084            tokio::fs::create_dir_all(parent).await.ok();
1085        }
1086        let path = worktree.to_string_lossy().to_string();
1087        git::git(repo, &["worktree", "add", &path, branch])
1088            .await
1089            .with_context(|| {
1090                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1091            })?;
1092
1093        let setup = match ensure_setup_config(&mut state, repo).await {
1094            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1095            Err(e) => Err(e),
1096        };
1097        if let Err(e) = setup {
1098            git::worktree_remove_clean(repo, &worktree).await.ok();
1099            return Err(e);
1100        }
1101        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1102            .await
1103            .unwrap_or(0);
1104        if commits == 0 {
1105            git::worktree_remove_clean(repo, &worktree).await.ok();
1106            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1107        }
1108        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1109            .await
1110            .map(|f| f.len())
1111            .unwrap_or(0);
1112        if files == 0
1113            && let (Ok(head_tree), Ok(base_tree)) = (
1114                git::tree_of(&worktree, "HEAD").await,
1115                git::tree_of(&worktree, &base_commit).await,
1116            )
1117            && head_tree == base_tree
1118        {
1119            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1120            git::worktree_remove_clean(repo, &worktree).await.ok();
1121            bail!(
1122                "`{branch}` at {} has a tree identical to base {}; this usually means \
1123                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1124                 against `{}/{branch}`) rather than an empty change",
1125                short(&head),
1126                short(&base_commit),
1127                state.config.merge.remote
1128            );
1129        }
1130        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1131            .await
1132            .unwrap_or_default();
1133
1134        state.candidates.push(Candidate {
1135            index: 0,
1136            label: 'A',
1137            // Not an agent id on purpose: nothing in the roster wrote this, and
1138            // the stats tables must not credit anyone with a win for it.
1139            agent: EXISTING_BRANCH.to_owned(),
1140            branch: branch.to_owned(),
1141            worktree,
1142            summary: String::new(),
1143            stat,
1144            files,
1145            commits,
1146            empty: false,
1147            failed: None,
1148            verified_noop: None,
1149            duration_ms: 0,
1150            folded: false,
1151        });
1152        state.tally = Some(Tally {
1153            first_choice: BTreeMap::from([('A', 0)]),
1154            borda: BTreeMap::new(),
1155            winner: 'A',
1156            rankings: 0,
1157            unanimous_initial: false,
1158            deliberated: false,
1159            changed_votes: 0,
1160            unanimous_final: false,
1161            tie_break: None,
1162            // No panel sat, so no quorum applies. Zero judges is the correct
1163            // number for work that never competed, and must not be reported as
1164            // a collapsed panel.
1165            judges: 0,
1166            present: 0,
1167            quorum: 0,
1168            met_quorum: true,
1169            uncontested: Some("review-only run: nothing competed".to_owned()),
1170        });
1171        state.status = RunStatus::Reviewing;
1172        state.prep_complete = Some(true);
1173        state.event(
1174            "start",
1175            format!(
1176                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1177                state.id
1178            ),
1179        );
1180        state.save()?;
1181        Ok(Self {
1182            state,
1183            roles,
1184            sem: Arc::new(Semaphore::new(max_parallel)),
1185            pause: Pause::new(),
1186            interrupt: Pause::new(),
1187        })
1188    }
1189
1190    /// Authorize exactly one additional rebase, under the caller's queue task
1191    /// claim and the same owner/execution locks used by run maintenance.
1192    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1193        Self::authorize_base_recovery_request(home, id, task, false).await
1194    }
1195
1196    /// Queue release uses its durable approval generation exactly once.
1197    pub(crate) async fn authorize_base_recovery_from_resumption(
1198        home: &Path,
1199        id: &str,
1200        task: &str,
1201    ) -> Result<()> {
1202        Self::authorize_base_recovery_request(home, id, task, true).await
1203    }
1204
1205    async fn authorize_base_recovery_request(
1206        home: &Path,
1207        id: &str,
1208        task: &str,
1209        resumption: bool,
1210    ) -> Result<()> {
1211        let _owner = crate::pr_head::owner_lock(home, id)?;
1212        let mut state = RunState::load_under(id, home)?;
1213        crate::pr_head::ensure_owner(&state)?;
1214        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1215        anyhow::ensure!(
1216            queued.runs.last().is_some_and(|run| run == id)
1217                && queued.status != crate::queue::TaskStatus::Done
1218                && !queued.fresh_start
1219                && queued.review_branch.is_none(),
1220            "run task ownership changed"
1221        );
1222        anyhow::ensure!(
1223            state
1224                .origin
1225                .as_ref()
1226                .and_then(|o| o.task.as_deref())
1227                .is_none_or(|owner| owner == task),
1228            "run belongs to another task"
1229        );
1230        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1231        anyhow::ensure!(
1232            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1233            "run is live or its ownership is unknown"
1234        );
1235        // Judge the previous driver's lease before acquiring our maintenance
1236        // lease. The owner lock excludes new driver claims across both steps.
1237        let _execution = crate::driver::maintenance(home, id)?;
1238        anyhow::ensure!(
1239            state.status.resumable()
1240                && state.released_to.is_none()
1241                && state.retired_to.is_none()
1242                && state.pr_taken_over_by.is_none(),
1243            "run ownership is final"
1244        );
1245        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1246        anyhow::ensure!(
1247            !state.pr_publication_pending
1248                && !crate::handover::landing_checkpoint(&state)
1249                && state.merge_decision.is_none(),
1250            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1251        );
1252        let winner = state.winner().context("run has no retained winner")?;
1253        anyhow::ensure!(
1254            git::is_clean(&winner.worktree).await?,
1255            "retained worktree has uncommitted changes"
1256        );
1257        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1258        anyhow::ensure!(
1259            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1260            "retained worktree does not match the winner branch"
1261        );
1262        state.record_base_sync_stop(&head);
1263        state.save()?;
1264        if resumption {
1265            anyhow::ensure!(
1266                crate::daemon::compatible_run_instruction(&state, &queued),
1267                "resumption instruction does not match the retained run"
1268            );
1269            let approval = queued
1270                .resumption
1271                .as_ref()
1272                .context("no operator resumption")?;
1273            anyhow::ensure!(
1274                approval.instruction == queued.instruction,
1275                "resumption approval does not match the current task instruction"
1276            );
1277            state.authorize_base_sync_resumption(task, approval)?;
1278        } else {
1279            state.authorize_base_sync(task)?;
1280        }
1281        state.event(
1282            "base-recovery",
1283            "owner authorized one additional base rebase; fresh verification required",
1284        );
1285        state.save()?;
1286        Ok(())
1287    }
1288
1289    /// Reopen an existing run.
1290    pub fn resume(id: &str) -> Result<Self> {
1291        let state = RunState::load(id)?;
1292        crate::pr_head::ensure_owner(&state)?;
1293        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1294            bail!(
1295                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1296            );
1297        }
1298        if let Some(to) = &state.released_to {
1299            bail!(
1300                "run {} cannot be resumed: its worktree was released to run {}",
1301                state.short(),
1302                crate::run::short_of(to)
1303            );
1304        }
1305        let roles = state.config.resolve_roles()?;
1306        if let Some(id) = &state.config.graph.selected_implementer
1307            && state.candidates.iter().any(|c| c.agent != *id)
1308        {
1309            bail!(
1310                "replacement implementer pin does not match the saved candidates; task must stay held"
1311            );
1312        }
1313        let max_parallel = state.config.graph.max_parallel.max(1);
1314        Ok(Self {
1315            state,
1316            roles,
1317            sem: Arc::new(Semaphore::new(max_parallel)),
1318            pause: Pause::new(),
1319            interrupt: Pause::new(),
1320        })
1321    }
1322
1323    /// Walk the graph to a terminal state, skipping nodes already recorded.
1324    ///
1325    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1326    /// the phone - ends here, so this is the one place a run that ended
1327    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1328    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1329    pub async fn execute(&mut self) -> Result<()> {
1330        if let Some(selection) = &self.state.pr_recovery
1331            && (selection.source_proof.is_some() || selection.source_run.is_some())
1332        {
1333            Box::pin(selection.verify_source(&self.state.repo)).await?;
1334        }
1335        // Queue claims cover daemon work; this also serializes manual drivers
1336        // consuming a repair answer, including the new review and decision.
1337        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1338            || self.state.land_approval.is_some()
1339            || self.state.merge_decision.is_some()
1340            || !self.state.merge_repairs.is_empty()
1341            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1342        {
1343            let claim = FixClaim::acquire(&self.state.dir())?;
1344            let disk = RunState::load(&self.state.id)?;
1345            let checkpoint = |s: &RunState| {
1346                (
1347                    s.merge_repairs.len(),
1348                    s.merge_repairs
1349                        .last()
1350                        .map(|r| (r.question.clone(), r.phase)),
1351                    s.land_approval
1352                        .as_ref()
1353                        .map(|a| (a.question.clone(), a.head.clone())),
1354                )
1355            };
1356            if checkpoint(&disk) != checkpoint(&self.state) {
1357                bail!(
1358                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1359                );
1360            }
1361            Some(claim)
1362        } else {
1363            None
1364        };
1365        let mut execution = self.claim_driver().await?;
1366        let runner = &mut *execution.runner;
1367        let wait = runner.repository_wait();
1368        // Keep the graph's large future off the callers' stack, including
1369        // standalone task attempts running on a default-sized test thread.
1370        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1371        let result = match result {
1372            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1373                record_repository_block(&mut runner.state, &e);
1374                runner.park_here()?;
1375                Ok(())
1376            }
1377            other => other,
1378        };
1379        execution.driver.finish(&mut runner.state).await?;
1380        let ended = if result.is_err() {
1381            Some({
1382                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1383                if let Err(error) = &result {
1384                    notice.message = format!(
1385                        "Run {} stopped with an error: {}",
1386                        runner.state.id,
1387                        error.to_string().lines().next().unwrap_or("unknown error")
1388                    );
1389                }
1390                notice
1391            })
1392        } else {
1393            crate::notices::run_ended(&runner.state)
1394        };
1395        if let Some(notice) = ended {
1396            crate::notices::raise(notice);
1397        }
1398        result
1399    }
1400
1401    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1402        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1403    }
1404
1405    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1406        let wait = self.repository_wait();
1407        coordinate_operation(&mut self.state, cwd, &wait).await
1408    }
1409
1410    /// Validate ownership and persist the driver claim in one critical section.
1411    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1412        let owner = match crate::pr_head::owner_lock_wait(
1413            &crate::run::home(),
1414            &self.state.id,
1415            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1416        )
1417        .await
1418        {
1419            Ok(owner) => owner,
1420            Err(error) => {
1421                let detail = format!("worktree maintenance or ownership update: {error:#}");
1422                // Admission did not own run.json: retain this resource outcome
1423                // in memory for queue settlement without overwriting a concurrent
1424                // retirement or publication transaction.
1425                self.state.resource_block = Some(crate::run::ResourceBlock {
1426                    cause: crate::run::ResourceBlockCause::Other,
1427                    node: "start".into(),
1428                    kind: "worktree maintenance".into(),
1429                    detail: detail.clone(),
1430                    at: Timestamp::now(),
1431                });
1432                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1433            }
1434        };
1435        let disk = RunState::load(&self.state.id)?;
1436        crate::pr_head::ensure_owner(&disk)?;
1437        let claimed =
1438            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1439        // Only proven-live ownership refuses here. Unknown records (legacy runs
1440        // with no claim marker or process identity) fall through to the
1441        // exclusive execution lease taken just below, which is what actually
1442        // keeps a second driver out.
1443        anyhow::ensure!(
1444            disk.liveness(claimed) != crate::run::Liveness::Live,
1445            "run already has a live driver; wait for its execution to end"
1446        );
1447        if let Some(to) = &disk.released_to {
1448            bail!(
1449                "run {} cannot continue: its worktree was released to run {}",
1450                self.state.short(),
1451                crate::run::short_of(to)
1452            );
1453        }
1454        anyhow::ensure!(
1455            serde_json::to_value(&disk.base_sync_recovery)?
1456                == serde_json::to_value(&self.state.base_sync_recovery)?,
1457            "base recovery changed since this run was loaded; reload the saved run"
1458        );
1459        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1460        let mut execution = DriverExecution {
1461            runner: self,
1462            driver,
1463        };
1464        // Declare after the execution guard: on any claim error this short
1465        // lock must drop before that guard reloads and releases ownership.
1466        let _owner = owner;
1467        let state = &mut execution.runner.state;
1468        // Moving again, so it is no longer parked. Set before the walk rather
1469        // than in `resume`, so every way of re-entering the graph clears it
1470        // and a card cannot claim a run is waiting to be resumed while the
1471        // agents are already working.
1472        state.parked = false;
1473        state.resource_wait = None;
1474        state.resource_block = None;
1475        // Any seat this state still lists as answering belongs to whatever
1476        // process last drove this run — this one included, if it crashed
1477        // mid-wave. Cleared and flushed immediately, before anything else
1478        // runs, so a resume can never show a seat as live when nothing is
1479        // asking it anything yet; the node that actually dispatches the next
1480        // wave repopulates it.
1481        state.clear_active();
1482        // Recorded in the same spot, and flushed together with the clear
1483        // above: this is the pid a reader checks (`RunState::liveness`) when
1484        // no daemon claim exists to answer "is a process still driving this
1485        // run" — a plain `magi run` / `magi review` typed into a terminal
1486        // claims nothing there. Always overwritten, never only-if-absent, so
1487        // a resumed run's stale pid from a previous, possibly-dead process
1488        // can never survive into this one's own report. Unlike
1489        // `clear_active`, this changes on every single `execute()` call, so
1490        // the save below is now unconditional rather than only-if-cleared.
1491        //
1492        // `driver_started_at` is recorded in the same breath, from this same
1493        // pid, so `liveness` can tell a live pid that is genuinely still us
1494        // apart from one the OS has since handed to an unrelated process —
1495        // see that field's own doc for why the pid alone is not enough.
1496        let pid = std::process::id();
1497        state.driver_pid = Some(pid);
1498        state.driver_started_at = crate::proc::process_started_at(pid);
1499        state.driver_exited = false;
1500        state.driver_claimed = Some(true);
1501        state.begin_pass();
1502        execution.driver.bind(state);
1503        state.save()?;
1504        execution.driver.record(state)?;
1505        Ok(execution)
1506    }
1507
1508    async fn execute_graph(&mut self) -> Result<()> {
1509        self.state.configure_recovery_resume()?;
1510        // Preservation may invalidate an old empty classification. Resolve it
1511        // before any resume shortcut can reuse a verdict or publish a branch.
1512        self.persist_candidates().await?;
1513        // A saved base stop outranks stage markers, including legacy Voting.
1514        // Recovery never reenters implementation, deliberation or voting.
1515        if self.state.base_sync_blocked() {
1516            self.sync_to_base().await?;
1517            if !self.state.base_sync_blocked()
1518                && self.state.status != RunStatus::AlreadyInBase
1519                && !self.resource_halt()?
1520            {
1521                self.finish_after_tally().await?;
1522            }
1523            return Ok(());
1524        }
1525        if self
1526            .state
1527            .merge_decision
1528            .as_ref()
1529            .is_some_and(|decision| !decision.closed)
1530            && self.state.status != RunStatus::Landing
1531        {
1532            if crate::merge_decision::resumable_hold(&self.state) {
1533                self.state.merge = Some(MergeOutcome {
1534                    mode: MergeMode::Pr,
1535                    ok: false,
1536                    detail: self
1537                        .state
1538                        .merge_decision
1539                        .as_ref()
1540                        .unwrap()
1541                        .target
1542                        .pr
1543                        .clone(),
1544                    empty: false,
1545                });
1546                self.run_land().await?;
1547                self.settle_questions();
1548                return Ok(());
1549            }
1550            bail!(
1551                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1552            );
1553        }
1554        if self.state.config.graph.selected_implementer.is_some()
1555            && self.state.status == RunStatus::Blocked
1556            && self.state.tally.is_none()
1557            && self.state.candidate_preservation.is_empty()
1558            && self.state.interrupted_implementations.is_empty()
1559        {
1560            bail!(
1561                "selected implementation is held; explicitly release the owning task to retry it"
1562            );
1563        }
1564
1565        if self.state.merge_repairs.last().is_some_and(|r| {
1566            matches!(
1567                r.phase,
1568                crate::run::MergeRepairPhase::Requested
1569                    | crate::run::MergeRepairPhase::Fixing
1570                    | crate::run::MergeRepairPhase::Reviewing
1571            )
1572        }) {
1573            if self.repair_merge_review().await? {
1574                self.run_land().await?;
1575            }
1576            self.settle_questions();
1577            return Ok(());
1578        }
1579        // Publication already passed review and gate. Resume only that node:
1580        // syncing to a newer base here could change the verified commit.
1581        if self.state.pr_publication_pending {
1582            self.merge().await?;
1583            if self.resource_halt()? {
1584                return Ok(());
1585            }
1586            self.state.save()?;
1587            self.settle_questions();
1588            return Ok(());
1589        }
1590        // A run that already lost its quorum never resumes into the verdict
1591        // machinery: `deliberate` and `vote` would otherwise clobber the
1592        // stalled marker back to Voting and the run would keep going past a
1593        // verdict that is no longer trustworthy. Everything already recorded is
1594        // kept, so the run stays resumable (or foldable) for a human to pick up.
1595        //
1596        // On --resume the run gets one chance to repair itself: the seats a
1597        // rate limit took out are re-asked. If their quota has since reset and
1598        // the quorum is restored, the run picks up and finishes; otherwise it
1599        // stays stale and still-resumable for a later retry. If it does not
1600        // recover, the returned status stays `Stalled` and nothing was
1601        // clobbered (the recovery only mutates entries for the lost seats).
1602        if self.state.status == RunStatus::Stalled {
1603            if self.recover_stall().await? {
1604                self.finish_after_tally().await?;
1605            } else {
1606                // Still below quorum: persist the marker and stay resumable.
1607                self.state.save()?;
1608            }
1609            return Ok(());
1610        }
1611        // A run parked inside `land` - watching CI, mid fix-round, or
1612        // waiting on the owner's merge approval - resumes directly into it,
1613        // never back through `prep`. Everything before `merge` already
1614        // concluded; that is the only way `status` reaches `Landing` in the
1615        // first place. Re-walking `review_loop` first would also be actively
1616        // wrong: its own status recomputation (see its doc) treats any
1617        // clean round as reason to set `status` to `Gating`, which would
1618        // clobber this marker before `merge` ever ran, and this run would
1619        // never find its way back into `land` at all.
1620        if self.state.status == RunStatus::Landing
1621            || crate::handover::landing_checkpoint(&self.state)
1622        {
1623            self.state.land_sync_pending = true;
1624            self.state.save()?;
1625            self.run_land().await?;
1626            // `run_land` may have settled the run right here - CI came back
1627            // green and the PR merged, say - without ever passing back
1628            // through `merge`'s own trailing call. Whatever it left `status`
1629            // as is what this has to read.
1630            self.settle_questions();
1631            return Ok(());
1632        }
1633        // A run parked at `merge` on the owner's word about a withheld PR text
1634        // resumes straight into `merge`. Re-walking the tail would sync to the
1635        // base again (possibly rebasing, possibly spending the fixer) after the
1636        // gate had passed, and then push a tree nobody reviewed or gated.
1637        if self.state.github_text.is_some()
1638            && self.state.merge.is_none()
1639            && self.state.status == RunStatus::Gating
1640        {
1641            self.merge().await?;
1642            if self.resource_halt()? {
1643                return Ok(());
1644            }
1645            self.state.save()?;
1646            return Ok(());
1647        }
1648        if let Err(e) = self.prep().await {
1649            if e.is::<crate::repo_lock::ResourceBlocked>() {
1650                return Err(e);
1651            }
1652            self.state.status = RunStatus::Blocked;
1653            self.state
1654                .prep_failure
1655                .get_or_insert_with(|| format!("{e:#}"));
1656            self.state
1657                .event("prep", format!("preparation stopped: {e:#}"));
1658            self.state.save()?;
1659            return Err(e);
1660        }
1661        if self.park_here()? {
1662            return Ok(());
1663        }
1664        self.advise().await?;
1665        if self.park_here()? {
1666            return Ok(());
1667        }
1668        self.implement().await?;
1669        if self.park_here()? {
1670            return Ok(());
1671        }
1672        // `after_implement` already saved the state and settled any open
1673        // questions when it set this; nothing later in the graph has
1674        // anything to judge.
1675        if self.state.status == RunStatus::VerifiedNoop {
1676            return Ok(());
1677        }
1678        self.judge().await?;
1679        if self.park_here()? {
1680            return Ok(());
1681        }
1682        self.deliberate().await?;
1683        if self.park_here()? {
1684            return Ok(());
1685        }
1686        self.vote().await?;
1687        if self.park_here()? {
1688            return Ok(());
1689        }
1690        self.tally()?;
1691        // A verdict that lost its quorum is not trustworthy: do not review,
1692        // gate, or merge on it. Everything already done is kept, so the run
1693        // stays resumable (or foldable); the human can replace the agent that
1694        // ran out of quota and pick it up.
1695        if self.state.status == RunStatus::Stalled {
1696            // Persist the stalled marker now — the normal end-of-execute save
1697            // below is below this early return, and without it a resumed run
1698            // would reload a pre-tally status and keep going.
1699            self.state.save()?;
1700            return Ok(());
1701        }
1702        self.finish_after_tally().await?;
1703        Ok(())
1704    }
1705
1706    /// A wave could not start its agents because a shared resource stayed busy
1707    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1708    /// resumable. Unstarted seats have no result; completed peers retain
1709    /// their saved candidate state.
1710    fn stop_for_resource(&mut self) -> Result<()> {
1711        // Resource acquisition observes cancellation inside a node. Complete
1712        // the same durable park as every other node boundary before returning.
1713        if self.pause.parked() || self.interrupt.parked() {
1714            self.park_here()?;
1715            return Ok(());
1716        }
1717        if self.state.resource_block.is_some() {
1718            self.state.status = RunStatus::Blocked;
1719        }
1720        self.state.save()
1721    }
1722
1723    /// Stop walking the graph if a node ended on a resource block.
1724    fn resource_halt(&mut self) -> Result<bool> {
1725        if self.state.resource_block.is_none() {
1726            return Ok(false);
1727        }
1728        self.stop_for_resource()?;
1729        Ok(true)
1730    }
1731
1732    /// Park here if asked to, recording it in the run's own timeline.
1733    ///
1734    /// Returns whether the caller should stop walking the graph. The state is
1735    /// saved either way by the node that just finished; this adds the event so
1736    /// the operator's card says why a run that is neither finished nor moving
1737    /// is sitting where it is.
1738    fn park_here(&mut self) -> Result<bool> {
1739        let parking = self.pause.parked() || self.interrupt.parked();
1740        if !parking && self.state.resource_block.is_some() {
1741            self.stop_for_resource()?;
1742            return Ok(true);
1743        }
1744        // Either handle asking is enough - see `Pause`'s own doc for why
1745        // they are never the same one. `interrupt` is checked second so a
1746        // reason it carries is preferred in the message below over a plain
1747        // shutdown park racing it at the same boundary.
1748        if !self.pause.parked() && !self.interrupt.parked() {
1749            return Ok(false);
1750        }
1751        if self.state.parked {
1752            return Ok(true);
1753        }
1754        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1755        if let Some(block) = &mut self.state.resource_block
1756            && matches!(
1757                block.cause,
1758                crate::run::ResourceBlockCause::Cancelled { .. }
1759            )
1760        {
1761            block.cause = crate::run::ResourceBlockCause::Cancelled {
1762                park: cause.clone(),
1763            };
1764        }
1765        let why = match cause.as_ref().map(ParkCause::label) {
1766            Some(reason) => format!(
1767                "parked after `{}` ({reason}) — resume to carry on from here",
1768                self.state.status.as_str()
1769            ),
1770            None => format!(
1771                "parked after `{}` — resume to carry on from here",
1772                self.state.status.as_str()
1773            ),
1774        };
1775        self.state.event("park", why);
1776        self.state
1777            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1778        self.state.parked = true;
1779        self.state.save()?;
1780        Ok(true)
1781    }
1782
1783    /// Only a new explicit operator approval may retry failed preparation.
1784    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1785        if self.state.prep_failure.is_none() {
1786            return Ok(());
1787        }
1788        let failed_at = self
1789            .state
1790            .events
1791            .iter()
1792            .rev()
1793            .find(|event| event.node == "prep")
1794            .map_or(self.state.updated_at, |event| event.at);
1795        anyhow::ensure!(
1796            approval_at > failed_at,
1797            "preparation retry requires a new explicit release after the failure"
1798        );
1799        self.state.prep_failure = None;
1800        self.state.prep_complete = Some(false);
1801        self.state
1802            .event("prep-retry", "operator approved retry of preparation");
1803        Ok(())
1804    }
1805
1806    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1807    pub fn on_pause(&mut self, pause: Pause) {
1808        self.pause = pause;
1809    }
1810
1811    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1812    /// scheduler asking this one run - and no other - to park so a task
1813    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1814    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1815    /// handle.
1816    pub fn watch_interrupt(&mut self, pause: Pause) {
1817        self.interrupt = pause;
1818    }
1819
1820    /// Abandon this run's own open questions, once `status` has actually
1821    /// settled rather than merely paused.
1822    ///
1823    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1824    /// either back up with the candidates, the review round and the seat
1825    /// sessions already on disk, so a question an implementer asked mid-round
1826    /// may still get a real answer read by a real resume. Only the statuses
1827    /// `resumable` excludes are actually final: the run merged, it reached
1828    /// `Ready` with nothing left to do, it failed outright with no
1829    /// established point to continue from, or every candidate agreed, with
1830    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1831    /// every one of those the seat that asked is gone for good, exactly like
1832    /// the run being deleted under `magi run rm` - so the same cleanup
1833    /// applies, worded for what actually happened instead of "the run was
1834    /// deleted".
1835    ///
1836    /// Best-effort and silent on success: called from every place `status`
1837    /// can land on one of those three, including ones a resumed run revisits,
1838    /// so it must cost nothing when there was nothing open to begin with.
1839    fn settle_questions(&mut self) {
1840        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1841            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1842        }
1843    }
1844
1845    /// The tail of the graph after a trustworthy tally: fold losers, review,
1846    /// gate, merge, and persist.
1847    async fn finish_after_tally(&mut self) -> Result<()> {
1848        // Before the losers are folded: their branches are what the graft
1849        // reads from.
1850        self.graft().await?;
1851        if self.resource_halt()? {
1852            return Ok(());
1853        }
1854        self.fold_losers().await?;
1855        // Before review starts, and again right before the gate: a run's
1856        // review rounds can themselves take long enough for the base to move
1857        // a second time, and the gate is the one node whose "green" gets
1858        // acted on.
1859        self.sync_to_base().await?;
1860        if self.hold_base_sync().await? || self.resource_halt()? {
1861            return Ok(());
1862        }
1863        if self.state.status == RunStatus::AlreadyInBase {
1864            return Ok(());
1865        }
1866        self.review_loop().await?;
1867        if self.resource_halt()? {
1868            return Ok(());
1869        }
1870        self.sync_to_base().await?;
1871        if self.hold_base_sync().await? || self.resource_halt()? {
1872            return Ok(());
1873        }
1874        if self.state.status == RunStatus::AlreadyInBase {
1875            return Ok(());
1876        }
1877        self.gate().await?;
1878        self.merge().await?;
1879        if self.resource_halt()? {
1880            return Ok(());
1881        }
1882        self.state.save()?;
1883        Ok(())
1884    }
1885
1886    // ---------------------------------------------------------------- prep
1887
1888    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1889    /// run with the step and its output in the event; it is never swallowed.
1890    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1891    /// resume starts over instead of finding candidates and missing seats.
1892    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1893        let result = match ensure_setup_config(&mut self.state, repo).await {
1894            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1895            Err(e) => Err(e),
1896        };
1897        if let Err(e) = result {
1898            git::worktree_remove_clean(repo, wt).await.ok();
1899            if in_prep {
1900                self.rollback_prep(repo).await;
1901            }
1902            self.state.status = RunStatus::Blocked;
1903            self.state.event(
1904                "setup",
1905                format!("worktree setup failed in {}: {e:#}", wt.display()),
1906            );
1907            self.state.save()?;
1908            return Err(e);
1909        }
1910        Ok(())
1911    }
1912
1913    /// Undo everything `prep` created, so a resume re-runs it whole.
1914    async fn rollback_prep(&mut self, repo: &Path) {
1915        let root = self.state.worktree_root();
1916        for c in std::mem::take(&mut self.state.candidates) {
1917            if matches!(
1918                git::worktree_remove_clean(repo, &c.worktree).await,
1919                Ok(true)
1920            ) {
1921                git::branch_delete(repo, &c.branch).await.ok();
1922            } else {
1923                self.state.candidates.push(c);
1924            }
1925        }
1926        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1927        for j in 1..=seats {
1928            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1929                .await
1930                .ok();
1931        }
1932        for k in 1..=self.state.config.graph.advisors {
1933            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1934                .await
1935                .ok();
1936        }
1937    }
1938
1939    async fn prep(&mut self) -> Result<()> {
1940        if let Some(reason) = &self.state.prep_failure {
1941            bail!(
1942                "candidate preparation previously failed: {reason}; correct the source before retrying"
1943            );
1944        }
1945        if self
1946            .state
1947            .prep_complete
1948            .unwrap_or(!self.state.candidates.is_empty())
1949        {
1950            return Ok(());
1951        }
1952        let repo = self.state.repo.clone();
1953        // Includes rollback, config, refs, checkout creation and setup. No
1954        // repository mutation may precede the real shared lock acquisition.
1955        let _coordination = self.coordinate(&repo).await?;
1956        let base = self.state.base_commit.clone();
1957        let plan = self.state.sources.plan(&repo, &base).await?;
1958        self.rollback_prep(&repo).await;
1959        if !self.state.candidates.is_empty() {
1960            bail!(
1961                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1962            );
1963        }
1964        self.state.prep_complete = Some(false);
1965        self.state.status = RunStatus::Prep;
1966        self.state.save()?;
1967        let repo = self.state.repo.clone();
1968        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1969        let root = self.state.worktree_root();
1970        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1971
1972        // The hook is the write-time half of the blindness contract; the
1973        // presentation filter in `blind` is the half that cannot be bypassed.
1974        let hooks_dir = self.state.dir().join("hooks");
1975        if self.state.config.blind.commit_msg_hook {
1976            std::fs::create_dir_all(&hooks_dir)
1977                .with_context(|| format!("create {}", hooks_dir.display()))?;
1978            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1979            let path = hooks_dir.join("commit-msg");
1980            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1981            make_executable(&path)?;
1982            // Ref-counted rather than a plain idempotent set: with more than
1983            // one run able to be in flight in the same repository at once
1984            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1985            // true?" check cannot tell "another run of mine still needs
1986            // this" from "nobody does", and the run that happens to finish
1987            // first would disable the hook out from under a sibling still
1988            // relying on it.
1989            git::acquire_worktree_config(&repo).await?;
1990            self.state.enabled_worktree_config = true;
1991        }
1992
1993        // `[worktree] setup` hides its products per worktree, which needs the
1994        // same per-worktree config the hook does. Held until fold, like it.
1995        ensure_setup_config(&mut self.state, &repo).await?;
1996
1997        for (index, (spec, label)) in self
1998            .roles
1999            .implementers
2000            .clone()
2001            .into_iter()
2002            .zip(labels)
2003            .enumerate()
2004        {
2005            let branch = self.state.branch_for(label);
2006            let worktree = root.join(format!("cand-{label}"));
2007            // Persist intent before Git can leave a partially created tree.
2008            // Candidate registration follows setup, so it cannot inventory
2009            // crashes or refusals during creation itself.
2010            self.state.event(
2011                "prep-intent",
2012                serde_json::json!({
2013                    "branch": branch,
2014                    "worktree": worktree,
2015                    "start": start,
2016                    "picks": plan.picks,
2017                })
2018                .to_string(),
2019            );
2020            self.state.save()?;
2021            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2022            if self.state.config.blind.commit_msg_hook {
2023                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2024            }
2025            git::local_exclude(&worktree, "/.magi/").await?;
2026            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2027                // Nothing of this prep may survive: a resume must start over,
2028                // not find half the candidates and skip the rest.
2029                git::worktree_remove_clean(&repo, &worktree).await.ok();
2030                git::branch_delete(&repo, &branch).await.ok();
2031                self.rollback_prep(&repo).await;
2032                self.state.status = RunStatus::Blocked;
2033                self.state
2034                    .event("prep", format!("worktree setup failed: {e:#}"));
2035                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2036                self.state.save()?;
2037                return Err(e);
2038            }
2039            self.state.candidates.push(Candidate {
2040                index,
2041                label,
2042                agent: spec.id.clone(),
2043                branch,
2044                worktree,
2045                summary: String::new(),
2046                stat: String::new(),
2047                files: 0,
2048                commits: 0,
2049                empty: false,
2050                failed: None,
2051                verified_noop: None,
2052                duration_ms: 0,
2053                folded: false,
2054            });
2055            self.state.save()?;
2056            for pick in &plan.picks {
2057                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2058                {
2059                    self.state.status = RunStatus::Blocked;
2060                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2061                    self.state
2062                        .event("prep", format!("cannot apply referenced commit: {e}"));
2063                    self.state.save()?;
2064                    return Err(e);
2065                }
2066            }
2067        }
2068
2069        for j in 1..=self.roles.judges.len() {
2070            let wt = root.join(format!("judge-{j}"));
2071            if !wt.exists() {
2072                git::worktree_add_detached(&repo, &wt, &base).await?;
2073                self.setup_seat_worktree(&repo, &wt, true).await?;
2074            }
2075        }
2076
2077        // Disposable, detached checkouts for the design-deliberation stage's
2078        // advisor seats — the same shape as the judges' above, at the same
2079        // base commit, since advisors also only ever read. Sized off the
2080        // configured count directly rather than a resolved roster: unlike
2081        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2082        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2083        // `prep` has no `ResolvedRoles` field to read a count from here.
2084        if self.state.config.graph.advise {
2085            for k in 1..=self.state.config.graph.advisors {
2086                let wt = root.join(format!("advisor-{k}"));
2087                if !wt.exists() {
2088                    git::worktree_add_detached(&repo, &wt, &base).await?;
2089                    self.setup_seat_worktree(&repo, &wt, true).await?;
2090                }
2091            }
2092        }
2093
2094        // A judge cannot tell it is looking at its own patch — the seats keep
2095        // separate conversations — but a panel that shares agents with the
2096        // field is less independent than it looks, and that is worth saying out
2097        // loud once per run rather than leaving it in the config.
2098        let authors: Vec<&str> = self
2099            .roles
2100            .implementers
2101            .iter()
2102            .map(|a| a.id.as_str())
2103            .collect();
2104        let overlap: Vec<String> = self
2105            .roles
2106            .judges
2107            .iter()
2108            .enumerate()
2109            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2110            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2111            .collect();
2112        if !overlap.is_empty() {
2113            let note = format!(
2114                "{} also authored a candidate; blind, but the panel is less \
2115                 independent than {} distinct agents would be",
2116                overlap.join(", "),
2117                self.roles.judges.len()
2118            );
2119            self.state.event("prep", note);
2120        }
2121
2122        self.state.event(
2123            "prep",
2124            format!(
2125                "{} candidates, {} judges, base {} ({})",
2126                self.state.candidates.len(),
2127                self.roles.judges.len(),
2128                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2129                self.state.base_branch
2130            ),
2131        );
2132        self.state.prep_complete = Some(true);
2133        self.state.status = RunStatus::Implementing;
2134        self.state.save()?;
2135        Ok(())
2136    }
2137
2138    // -------------------------------------------------------------- advise
2139
2140    /// The design-deliberation stage: independent, read-only advisor seats
2141    /// each sketch a design before any implementer touches the repository,
2142    /// and (when at least one produced a usable proposal) a synthesis seat
2143    /// blends them into a brief `implement` carries in every candidate's
2144    /// prompt.
2145    ///
2146    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2147    /// advisors` is the proposal count. Everything here is best-effort and
2148    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2149    /// that cannot reach quota, or a synthesis seat that produced nothing
2150    /// usable all leave `implement` exactly as it was before this stage
2151    /// existed — the task instruction alone — rather than failing the whole
2152    /// competition over an enrichment stage. Every outcome is still recorded
2153    /// as an event, so a run that got nothing from this stage says why.
2154    ///
2155    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2156    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2157    /// resumed run whose stage failed would re-run it, and re-spend the
2158    /// agent calls, on every reentry before `implement`.
2159    ///
2160    /// Also skipped once any candidate shows implementation progress — the
2161    /// exact predicate `implement` itself uses to decide a candidate is no
2162    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2163    /// is not enough: a run created by an older binary that predates this
2164    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2165    /// an already-`Implementing`-or-later run under this build would
2166    /// otherwise walk straight back through `prep` (a no-op once candidates
2167    /// exist) into this node and spawn every advisor seat against worktrees
2168    /// `prep` never recreated — after implementation has already started,
2169    /// which is exactly the invariant this stage exists to guarantee.
2170    async fn advise(&mut self) -> Result<()> {
2171        let implement_untouched = self
2172            .state
2173            .candidates
2174            .iter()
2175            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2176        if !self.state.config.graph.advise || self.state.advise_attempted {
2177            return Ok(());
2178        }
2179        if !implement_untouched {
2180            self.state.event(
2181                "advise",
2182                "skipping the design-deliberation stage: at least one \
2183                 candidate already shows implementation progress, so this \
2184                 run is past the point the stage exists to run before"
2185                    .to_owned(),
2186            );
2187            self.state.advise_attempted = true;
2188            self.state.save()?;
2189            return Ok(());
2190        }
2191        let run_id = self.state.id.clone();
2192        let prompts = self.state.config.prompts.clone();
2193        let instruction = self.state.instruction.clone();
2194        let language = self.state.config.graph.language.clone();
2195        let root = self.state.worktree_root();
2196        let n = self.state.config.graph.advisors;
2197        let where_recorded = self.state.dir().join("run.json");
2198
2199        let seats = match self.state.config.advisors() {
2200            Ok(seats) if !seats.is_empty() => seats,
2201            Ok(_) => {
2202                self.state.event(
2203                    "advise",
2204                    format!(
2205                        "[graph] advisors is 0; skipping the design-deliberation \
2206                         stage and continuing without a synthesis brief (see {})",
2207                        where_recorded.display()
2208                    ),
2209                );
2210                self.state.advise_attempted = true;
2211                self.state.save()?;
2212                return Ok(());
2213            }
2214            Err(e) => {
2215                self.state.event(
2216                    "advise",
2217                    format!(
2218                        "could not resolve advisor seats ({e:#}); continuing \
2219                         without a design-deliberation brief (see {})",
2220                        where_recorded.display()
2221                    ),
2222                );
2223                self.state.advise_attempted = true;
2224                self.state.save()?;
2225                return Ok(());
2226            }
2227        };
2228
2229        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2230        let artifacts = agent::artifacts_dir(&self.state.dir());
2231        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2232
2233        let mut jobs = Vec::new();
2234        for (i, spec) in seats.iter().cloned().enumerate() {
2235            let seat_key = format!("advisor-{}", i + 1);
2236            let seat = self.seat(&seat_key, &spec.id);
2237            jobs.push(SeatJob {
2238                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2239                spec,
2240                seat,
2241                cwd: worktrees[i % worktrees.len()].clone(),
2242                timeout,
2243                allow_write: false,
2244                sessions: false,
2245                artifacts: artifacts.clone(),
2246                stem: seat_key,
2247                handover: None,
2248            });
2249        }
2250
2251        self.state.event(
2252            "advise",
2253            format!(
2254                "{} advisor seat(s) sketching a design in parallel",
2255                jobs.len()
2256            ),
2257        );
2258        let mut quota_losses = Vec::new();
2259        let cache = self.state.config.cache_dir();
2260        let ctx = WaveCtx {
2261            carry_seats: false,
2262            pauses: Some((&self.pause, &self.interrupt)),
2263            run: &run_id,
2264            node: "advise",
2265            prompts: &prompts,
2266            cache: cache.as_deref(),
2267            round: None,
2268        };
2269        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2270        let results = ask_json_wave::<Proposal>(
2271            jobs,
2272            Arc::clone(&self.sem),
2273            self.state.config.graph.retries,
2274            &advisor_roster,
2275            &ctx,
2276            &mut quota_losses,
2277            &mut self.state,
2278            &|p: &Proposal| p.validate(),
2279        )
2280        .await;
2281        if self.state.resource_block.is_some() {
2282            return self.stop_for_resource();
2283        }
2284        self.state.quota.extend(quota_losses);
2285
2286        let mut records = Vec::with_capacity(results.len());
2287        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2288            let agent_id = seat.agent.clone();
2289            self.state.seats.insert(seat.key.clone(), seat);
2290            match res {
2291                Ok((proposal, out)) => {
2292                    self.state
2293                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2294                    records.push(advise::AdvisorRecord::proposed(
2295                        i + 1,
2296                        agent_id,
2297                        proposal,
2298                        out.duration_ms,
2299                    ));
2300                }
2301                Err(e) => {
2302                    self.state.event(
2303                        "advise",
2304                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2305                    );
2306                    records.push(advise::AdvisorRecord::failed(
2307                        i + 1,
2308                        agent_id,
2309                        e.to_string(),
2310                    ));
2311                }
2312            }
2313        }
2314
2315        let mut advice = advise::Advice {
2316            records,
2317            synthesis: None,
2318        };
2319        if advice.proposals().is_empty() {
2320            self.state.event(
2321                "advise",
2322                "no advisor produced a usable proposal; continuing without a \
2323                 synthesis brief"
2324                    .to_owned(),
2325            );
2326        } else {
2327            match self
2328                .synthesize_brief(
2329                    &advice,
2330                    &instruction,
2331                    &language,
2332                    &worktrees[0],
2333                    &artifacts,
2334                    &run_id,
2335                    &prompts,
2336                    cache.as_deref(),
2337                )
2338                .await
2339            {
2340                Ok(Some(text)) => {
2341                    self.state.event(
2342                        "advise",
2343                        "synthesized a design brief for the implementer".to_owned(),
2344                    );
2345                    advice.synthesis = Some(text);
2346                }
2347                Ok(None) => {
2348                    self.state.event(
2349                        "advise",
2350                        "the synthesis seat produced nothing usable; continuing \
2351                         without a design brief"
2352                            .to_owned(),
2353                    );
2354                }
2355                Err(e) => {
2356                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2357                        return Err(e);
2358                    }
2359                    self.state.event(
2360                        "advise",
2361                        format!("could not synthesize a design brief: {e:#}"),
2362                    );
2363                }
2364            }
2365        }
2366        advise::apply_reflection(&mut advice);
2367
2368        self.state.advice = Some(advice);
2369        self.state.advise_attempted = true;
2370        self.state.save()?;
2371        Ok(())
2372    }
2373
2374    /// The synthesis seat: reads every advisor's proposal and blends them
2375    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2376    /// out of [`Runner::advise`] only for readability — it is not called
2377    /// anywhere else.
2378    ///
2379    /// Picked the same way [`crate::talk`]'s standing conversation and
2380    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2381    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2382    /// order (a claude seat, else the first runnable agent in roster order)
2383    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2384    /// in [`crate::config`] for why a dedicated field exists here at all.
2385    #[allow(clippy::too_many_arguments)]
2386    async fn synthesize_brief(
2387        &mut self,
2388        advice: &advise::Advice,
2389        instruction: &str,
2390        language: &str,
2391        cwd: &Path,
2392        artifacts: &Path,
2393        run_id: &str,
2394        prompts: &Prompts,
2395        cache: Option<&Path>,
2396    ) -> Result<Option<String>> {
2397        let chain = agent::pick_chain(
2398            &self.state.config.agents,
2399            self.state.config.roles.synthesizer.as_ref(),
2400            &agent::installed,
2401            "synthesizer",
2402        )?;
2403        let proposals = advice.proposals();
2404        let mut prompt = prompt::with_overlay(
2405            prompt::synthesize_brief(instruction, &proposals, language),
2406            prompts.overlay("advise"),
2407        );
2408        if cache.is_some() {
2409            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2410            // below — see `prompt::build_cache_note`'s doc for why telling a
2411            // read-only seat to build through the shared cache is exactly how
2412            // a sandbox's write refusal gets misread as a defect.
2413            prompt.push('\n');
2414            prompt.push_str(&prompt::build_cache_note("advise", false));
2415        }
2416        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2417        // Each id is tried once, in order; a quota hit, error or unusable
2418        // answer moves to the next. The seat is single-turn (`sessions:
2419        // false`) and the prompt is the whole context, so a fallback agent
2420        // needs nothing carried over.
2421        let mut last = None;
2422        for (n, spec) in chain.iter().enumerate() {
2423            if n > 0 {
2424                self.state
2425                    .event("advise", format!("synthesis falling back to {}", spec.id));
2426            }
2427            let mut seat = self.seat("advise-synthesis", &spec.id);
2428            let wait = self.repository_wait();
2429            let outcome = wait
2430                .scope(agent::invoke(
2431                    spec,
2432                    &mut seat,
2433                    &Invocation {
2434                        coordination: self.state.config.coordination.as_ref(),
2435                        fetched_base: None,
2436                        cwd,
2437                        prompt: &prompt,
2438                        timeout,
2439                        allow_write: false,
2440                        sessions: false,
2441                        artifacts,
2442                        stem: &if n == 0 {
2443                            "advise-synthesis".to_owned()
2444                        } else {
2445                            format!("advise-synthesis-{}", spec.id)
2446                        },
2447                        run: run_id,
2448                        node: "advise",
2449                        cache_dir: None,
2450                        attachments: &[],
2451                        writable: &[],
2452                    },
2453                ))
2454                .await;
2455            if outcome.is_ok() {
2456                self.state.seats.insert(seat.key.clone(), seat);
2457            }
2458            let advance = agent::chain_advances(&outcome);
2459            last = Some(outcome);
2460            if !advance {
2461                break;
2462            }
2463        }
2464        // Exhausted: the last attempt's result is what a single failed seat
2465        // would have produced.
2466        let out = last.expect("a chain holds at least one agent")?;
2467        if !out.usable() {
2468            return Ok(None);
2469        }
2470        let text =
2471            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2472        Ok((!text.trim().is_empty()).then_some(text))
2473    }
2474
2475    // ----------------------------------------------------------- implement
2476
2477    async fn implement(&mut self) -> Result<()> {
2478        self.persist_candidates().await?;
2479        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2480        // agent files with `magi task add` name the run that paid for it. The
2481        // prompt overlay is cloned alongside it because the waves borrow it
2482        // while `self` is mutably borrowed by the node's own bookkeeping.
2483        let run_id = self.state.id.clone();
2484        let prompts = self.state.config.prompts.clone();
2485        let todo: Vec<usize> = self
2486            .state
2487            .candidates
2488            .iter()
2489            .enumerate()
2490            .filter(|(_, c)| {
2491                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2492                    && c.failed.is_none()
2493                    && !c.empty
2494            })
2495            .map(|(i, _)| i)
2496            .collect();
2497        if todo.is_empty() {
2498            return self.after_implement();
2499        }
2500        self.state.status = RunStatus::Implementing;
2501
2502        let language = self.state.config.graph.language.clone();
2503        let github_language = self.state.config.graph.github_language.clone();
2504        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2505        let sessions = self.state.config.graph.sessions;
2506        let artifacts = agent::artifacts_dir(&self.state.dir());
2507        // The design-deliberation stage's blended brief, when `advise` found
2508        // one — carried into every implementer's prompt the same way
2509        // regardless of which candidate it is.
2510        let brief = self
2511            .state
2512            .advice
2513            .as_ref()
2514            .and_then(|a| a.synthesis.as_deref())
2515            .map(str::to_owned);
2516        let attachments = self.state.attachments.clone();
2517
2518        let mut jobs = Vec::new();
2519        for &i in &todo {
2520            let (index, label, worktree) = {
2521                let c = &self.state.candidates[i];
2522                (c.index, c.label, c.worktree.clone())
2523            };
2524            let spec = self.roles.implementers[index].clone();
2525            let seat_key = format!("impl-{label}");
2526            let seat = self.seat(&seat_key, &spec.id);
2527            let instruction = seeded_instruction(&self.state);
2528            jobs.push(SeatJob {
2529                spec,
2530                seat,
2531                prompt: prompt::implement_in(
2532                    &instruction,
2533                    &worktree.to_string_lossy(),
2534                    &language,
2535                    &github_language,
2536                    brief.as_deref(),
2537                    &attachments,
2538                ),
2539                cwd: worktree,
2540                timeout,
2541                allow_write: true,
2542                sessions,
2543                artifacts: artifacts.clone(),
2544                stem: format!("impl-{label}"),
2545                handover: None,
2546            });
2547        }
2548
2549        self.state.event(
2550            "implement",
2551            format!("{} candidates in parallel", jobs.len()),
2552        );
2553        // Kept so a seat whose CLI hung up can be asked again from the same
2554        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2555        // can update a seat's own entry once a fallback agent takes it over —
2556        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2557        // whichever agent actually answered, not the one that quota'd out.
2558        let mut sent = jobs.clone();
2559        let cache = self.state.config.cache_dir();
2560        let ctx = WaveCtx {
2561            carry_seats: false,
2562            pauses: Some((&self.pause, &self.interrupt)),
2563            run: &run_id,
2564            node: "implement",
2565            prompts: &prompts,
2566            cache: cache.as_deref(),
2567            round: None,
2568        };
2569        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2570        if !wave_blocked(&results) {
2571            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2572                .await;
2573            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2574                .await?;
2575            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2576                .await;
2577        }
2578        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2579
2580        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2581            && results
2582                .iter()
2583                .any(|(_, _, out)| FailClass::of(out).is_some());
2584        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2585            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2586                // Executed turns interrupted by disk pressure already carry
2587                // a preservation barrier from wave. Admission-only waits do
2588                // not. Keep the interrupted seat's session for recovery too.
2589                if self
2590                    .state
2591                    .candidate_preservation
2592                    .contains_key(&self.state.candidates[i].label)
2593                {
2594                    self.state.seats.insert(seat.key.clone(), seat);
2595                }
2596                continue;
2597            }
2598            let seat_key = seat.key.clone();
2599            // A quota fallback (`resume_seat_handovers`) may have handed this
2600            // seat to a different agent than the one `prep` recorded on the
2601            // candidate; the stats tables and any later fixer-defaults-to-
2602            // winner's-author lookup must credit whoever actually answered —
2603            // unless every fallback also quota'd out, in which case nobody
2604            // actually answered and crediting the last agent tried would
2605            // erase every earlier agent's own quota loss from the stats
2606            // tables instead of just this one seat's.
2607            let agent = seat.agent.clone();
2608            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2609            self.state.seats.insert(seat.key.clone(), seat);
2610            let label = self.state.candidates[i].label;
2611            self.state.interrupted_implementations.remove(&label);
2612
2613            let (summary, duration, failed, verified_claim) = match out {
2614                AgentOutcome::Ok(o) => {
2615                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2616                    let failed = (!o.usable()).then(|| {
2617                        if o.timed_out {
2618                            "agent timed out".to_owned()
2619                        } else {
2620                            format!(
2621                                "agent exited with {}",
2622                                crate::notices::exit_status(o.exit_code)
2623                            )
2624                        }
2625                    });
2626                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2627                    (text, o.duration_ms, failed, verified_claim)
2628                }
2629                // Left un-resumed by `resume_undelivered` (a dirty tree
2630                // already rescues the work, or there was no session left to
2631                // resume into) — reported like the ordinary failure it is,
2632                // never as if `o.text` (the CLI's raw error JSON) were an
2633                // answer.
2634                AgentOutcome::Dropped(o) => {
2635                    let why = o
2636                        .dropped
2637                        .as_ref()
2638                        .map(|d| d.why.as_str())
2639                        .unwrap_or("the CLI ended the stream without delivering its answer");
2640                    (
2641                        String::new(),
2642                        o.duration_ms,
2643                        Some(format!("the CLI dropped the stream ({why})")),
2644                        None,
2645                    )
2646                }
2647                AgentOutcome::Quota(o) => {
2648                    self.state.quota.push(QuotaLoss {
2649                        seat: seat_key,
2650                        node: "implement".to_owned(),
2651                        at: Timestamp::now(),
2652                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2653                    });
2654                    (
2655                        String::new(),
2656                        o.duration_ms,
2657                        Some("rate limited (quota); produced no change".to_owned()),
2658                        None,
2659                    )
2660                }
2661                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2662                    (String::new(), 0, Some(e), None)
2663                }
2664            };
2665
2666            // Save every completed turn before any fallible Git operation. A failed
2667            // rescue must not erase the reply or prevent collection of other seats.
2668            let c = &mut self.state.candidates[i];
2669            if !exhausted_the_fallback_chain {
2670                c.agent = agent;
2671            }
2672            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2673            c.duration_ms = duration;
2674            c.failed = failed;
2675            c.verified_noop = verified_claim;
2676            self.state
2677                .candidate_preservation
2678                .insert(label, "awaiting persistence".to_owned());
2679        }
2680        self.state.save()?;
2681        self.persist_candidates().await?;
2682
2683        if resource_blocked {
2684            return self.stop_for_resource();
2685        }
2686        if selected_failed {
2687            self.state.status = RunStatus::Blocked;
2688            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2689            self.state.save()?;
2690            bail!(
2691                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2692            );
2693        }
2694        self.after_implement()
2695    }
2696
2697    /// Retry only persistence after a completed implementation turn.
2698    async fn persist_candidates(&mut self) -> Result<()> {
2699        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2700        for label in labels {
2701            let i = self
2702                .state
2703                .candidates
2704                .iter()
2705                .position(|c| c.label == label)
2706                .context("pending candidate is missing")?;
2707            let c = self.state.candidates[i].clone();
2708            let interrupted = self.state.interrupted_implementations.contains(&label);
2709            let result = async {
2710                let wait = crate::repo_lock::WaitPolicy::new(
2711                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2712                    || false,
2713                );
2714                let _coordination =
2715                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2716                // Best effort evidence requires no index writes. A successful snapshot
2717                // never authorizes removal of a tree whose rescue failed.
2718                if let Err(e) = git::snapshot_uncommitted(
2719                    &c.worktree,
2720                    &self
2721                        .state
2722                        .dir()
2723                        .join("artifacts")
2724                        .join(format!("candidate-{label}-recovery")),
2725                )
2726                .await
2727                {
2728                    self.state.event(
2729                        "preserve",
2730                        format!("candidate {label} snapshot incomplete: {e:#}"),
2731                    );
2732                }
2733                let rescued = git::rescue_commit(
2734                    &c.worktree,
2735                    &format!("magi: candidate {label} (uncommitted work)"),
2736                )
2737                .await?;
2738                self.state.note_withheld("implement", &rescued.withheld);
2739                let base = &self.state.base_commit;
2740                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2741                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2742                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2743                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2744                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2745                if c.empty && commits > 0 && !patch.trim().is_empty()
2746                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2747                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2748                        || self.state.tally.is_some())
2749                {
2750                    // Keep the old classification and pending marker so every
2751                    // retry stops too. Retain decisions as recovery evidence;
2752                    // only a supported recovery operation may replace them.
2753                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2754                }
2755                let c = &mut self.state.candidates[i];
2756                c.commits = commits;
2757                c.stat = stat;
2758                c.files = files;
2759                let no_changes = commits == 0 || patch.trim().is_empty();
2760                c.empty = no_changes && !interrupted;
2761                if interrupted && no_changes {
2762                    c.failed = None;
2763                    c.verified_noop = None;
2764                }
2765                if !c.empty {
2766                    c.failed = None;
2767                    c.verified_noop = None;
2768                }
2769                self.state.event(
2770                    "preserve",
2771                    format!("candidate {label}: {files} files, {commits} commits"),
2772                );
2773                Ok::<_, anyhow::Error>(())
2774            }
2775            .await;
2776            match result {
2777                Ok(()) => {
2778                    self.state.candidate_preservation.remove(&label);
2779                    if !interrupted || self.state.candidates[i].files > 0 {
2780                        self.state.interrupted_implementations.remove(&label);
2781                    }
2782                }
2783                Err(e) => {
2784                    let reason = format!("candidate {label} preservation failed: {e:#}");
2785                    self.state.event("preserve", &reason);
2786                    self.state.candidate_preservation.insert(label, reason);
2787                }
2788            }
2789            if self.state.candidate_preservation.is_empty()
2790                && self.state.status == RunStatus::Blocked
2791                && self.state.candidates.iter().all(|c| c.failed.is_none())
2792            {
2793                // Persist the recovered phase with the cleared pending marker;
2794                // a crash between saves must not turn this into a selected-seat
2795                // failure hold on the next resume.
2796                self.state.status = RunStatus::Implementing;
2797                self.state.event(
2798                    "preserve",
2799                    "candidate persistence recovered; implementation can continue",
2800                );
2801            }
2802            self.state.save()?;
2803        }
2804        if !self.state.candidate_preservation.is_empty() {
2805            self.state.status = RunStatus::Blocked;
2806            self.state.save()?;
2807            bail!(
2808                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2809            );
2810        }
2811        Ok(())
2812    }
2813
2814    /// Ask again, once, for work a CLI did and then failed to hand over.
2815    ///
2816    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2817    /// status with an empty response and a usage report showing output tokens,
2818    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2819    /// seven minutes and 14,267 output tokens that arrived as an empty
2820    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2821    ///
2822    /// Two conditions, and both matter:
2823    ///
2824    /// - **Only when the tree is untouched.** Often the agent has already
2825    ///   written its files and only the closing message was lost; the rescue
2826    ///   commit below picks that up and there is nothing to ask for. Re-asking
2827    ///   then would pay for a second implementation of work already on disk.
2828    /// - **Once.** A CLI that drops one stream can drop the next, and this
2829    ///   node is the most expensive in the graph.
2830    ///
2831    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2832    /// dropped reply still carried its `conversation_id`, so the seat is asked
2833    /// to finish what it was doing rather than sent the whole task again. It
2834    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2835    /// node's - for the same reason a re-ranked judge does: restating finished
2836    /// work is not the work.
2837    ///
2838    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2839    /// same way until it resets, while an abandoned conversation is still
2840    /// there to be picked up.
2841    async fn resume_undelivered(
2842        &mut self,
2843        results: &mut [(usize, SeatState, AgentOutcome)],
2844        sent: &[SeatJob],
2845        prompts: &Prompts,
2846        run_id: &str,
2847    ) {
2848        for (wi, seat, out) in results.iter_mut() {
2849            let Some(dropped) = (match &*out {
2850                AgentOutcome::Dropped(o) => o.dropped.clone(),
2851                _ => None,
2852            }) else {
2853                continue;
2854            };
2855            let Some(job) = sent.get(*wi) else { continue };
2856            // Already on disk? Then only the closing message was lost.
2857            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2858                self.state.event(
2859                    "implement",
2860                    format!(
2861                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2862                         work is in the tree",
2863                        seat.key, dropped.output_tokens, dropped.why
2864                    ),
2865                );
2866                continue;
2867            }
2868            // The re-ask only makes sense as a resume: `resume_after_drop`
2869            // says nothing about the task, trusting the seat to still hold it.
2870            // Without a session to resume — sessions disabled, or this CLI's
2871            // drop shape happened not to carry a session id — that prompt
2872            // would open a brand-new conversation with no context at all,
2873            // which is worse than leaving this as the ordinary failure it
2874            // already is.
2875            if !has_context(&job.spec, seat, job.sessions) {
2876                self.state.event(
2877                    "implement",
2878                    format!(
2879                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2880                         is no session left to resume",
2881                        seat.key, dropped.output_tokens, dropped.why
2882                    ),
2883                );
2884                continue;
2885            }
2886            self.state.event(
2887                "implement",
2888                format!(
2889                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2890                     conversation",
2891                    seat.key, dropped.output_tokens, dropped.why
2892                ),
2893            );
2894            let mut retry = job.clone();
2895            retry.seat = seat.clone();
2896            retry.prompt = prompt::resume_after_drop(&dropped.why);
2897            retry.timeout = retry_budget(job.timeout, true);
2898            retry.stem = format!("{}-resume", job.stem);
2899            let cache = self.state.config.cache_dir();
2900            let ctx = WaveCtx {
2901                carry_seats: false,
2902                pauses: Some((&self.pause, &self.interrupt)),
2903                run: run_id,
2904                node: "implement",
2905                prompts,
2906                cache: cache.as_deref(),
2907                round: None,
2908            };
2909            let (resumed_seat, resumed) =
2910                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2911            *seat = resumed_seat;
2912            *out = resumed;
2913        }
2914    }
2915
2916    /// Fall an implement seat through to the next untried agent in the
2917    /// implementer roster when it lost to quota — or, since the handover was
2918    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2919    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2920    /// quota path itself being unchanged — instead of leaving the
2921    /// seat's loss final the moment one agent's account runs dry.
2922    ///
2923    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2924    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2925    /// truncates to the single slot rotation picked, so a solo task whose one
2926    /// implementer hits quota mid-run used to have nothing else to try. This
2927    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2928    /// unrotated roster — which is the only place the *other* candidates in
2929    /// the machine's roster still exist once `implementers` has been cut down
2930    /// to size.
2931    ///
2932    /// Walks forward from just past the seat's own original position in the
2933    /// roster, never wrapping back to the front: a later candidate slot (say
2934    /// `beta`, the roster's second entry) must fall through to the *next*
2935    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2936    /// almost certainly a different candidate's own agent already — and once
2937    /// the roster's tail is exhausted there is nothing left to fall through
2938    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2939    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2940    /// let this retry that id forever. The loop keeps falling through until
2941    /// an attempt lands something other than `Quota` or the roster's tail
2942    /// runs out of untried ids, at which point the seat is left exactly as
2943    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2944    /// `QuotaLoss` recorded, the candidate failed/empty.
2945    ///
2946    /// `sent` is taken mutably and updated with the fallback agent's spec:
2947    /// `resume_unconfirmed_commands`, which runs after this and also reads
2948    /// `sent`, must see whichever agent actually ended up answering the seat
2949    /// — reading the stale, original spec there would check session
2950    /// eligibility against the wrong CLI and could hand a fallback agent's
2951    /// session id to the agent that just lost the seat to quota.
2952    ///
2953    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2954    /// — `self.seat` only reuses state when the agent id is unchanged, so
2955    /// handing it a different id already gets this for free. Reusing the old
2956    /// seat would resume a different CLI's session as if it were a
2957    /// continuation of this one.
2958    ///
2959    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2960    /// a quota loss cuts an agent off mid-turn, so anything already in the
2961    /// tree is unfinished work, not a completed candidate a re-ask would pay
2962    /// for twice. A dirty tree is rescued into a commit first (the same
2963    /// neutral-identity rescue `implement`'s own outcome loop gives every
2964    /// candidate) so the next agent starts clean.
2965    ///
2966    /// The new agent gets the implementer's full prompt and full
2967    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2968    /// it has no session and no context, and is implementing the task from
2969    /// nothing, unlike a resumed drop which is only restating work already
2970    /// done.
2971    ///
2972    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2973    /// `implement` event, never into `self.state.quota` — that is what
2974    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2975    /// go unspent, and a seat that ultimately recovered on its second or
2976    /// third agent is not the stalled panel that check exists to catch. Only
2977    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2978    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2979    /// outcome loop already has — this helper never pushes to it itself.
2980    async fn resume_seat_handovers(
2981        &mut self,
2982        results: &mut [(usize, SeatState, AgentOutcome)],
2983        sent: &mut [SeatJob],
2984        prompts: &Prompts,
2985        run_id: &str,
2986    ) -> Result<()> {
2987        let instruction = seeded_instruction(&self.state);
2988        let language = self.state.config.graph.language.clone();
2989        let github_language = self.state.config.graph.github_language.clone();
2990        let brief = self
2991            .state
2992            .advice
2993            .as_ref()
2994            .and_then(|a| a.synthesis.as_deref())
2995            .map(str::to_owned);
2996        let attachments = self.state.attachments.clone();
2997        for (wi, seat, out) in results.iter_mut() {
2998            // Who holds the other candidate seats of this wave right now
2999            // (earlier handovers already written back to `sent`).
3000            let others: BTreeSet<String> = sent
3001                .iter()
3002                .enumerate()
3003                .filter(|(j, _)| j != wi)
3004                .map(|(_, j)| j.spec.id.clone())
3005                .collect();
3006            let Some(job) = sent.get_mut(*wi) else {
3007                continue;
3008            };
3009            // Where the seat's own original agent sits in the roster — the
3010            // fallback walk starts just past here, never at the front, so a
3011            // later candidate slot's quota loss does not fall back onto an
3012            // earlier slot's own agent.
3013            let start = self
3014                .roles
3015                .implementer_roster
3016                .iter()
3017                .position(|s| s.id == job.spec.id)
3018                .unwrap_or(0);
3019            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3020            let mut fallback_attempt = 0usize;
3021            let mut prev: Option<FailClass> = None;
3022            while let Some(cur) = FailClass::of(&*out) {
3023                if !should_hand_over(prev.as_ref(), &cur) {
3024                    break;
3025                }
3026                let Some(next) =
3027                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3028                        .cloned()
3029                else {
3030                    break;
3031                };
3032                tried.insert(next.id.clone());
3033                fallback_attempt += 1;
3034
3035                let coordination = self.coordinate(&job.cwd).await?;
3036                if let Ok(r) = git::rescue_commit(
3037                    &job.cwd,
3038                    &format!(
3039                        "magi: candidate {} (uncommitted work before {} fallback)",
3040                        seat.key,
3041                        if cur == FailClass::Quota {
3042                            "quota"
3043                        } else {
3044                            "handover"
3045                        }
3046                    ),
3047                )
3048                .await
3049                {
3050                    self.state.note_withheld("implement", &r.withheld);
3051                }
3052                drop(coordination);
3053
3054                record_handover(
3055                    &mut self.state,
3056                    "implement",
3057                    &seat.key,
3058                    &seat.agent,
3059                    &next.id,
3060                    &cur,
3061                    &fail_reason(&*out),
3062                );
3063                prev = Some(cur.clone());
3064
3065                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3066                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3067                // Kept in sync on `sent` itself, not just the local retry: a
3068                // later helper (`resume_unconfirmed_commands`) reads `sent`
3069                // after this one returns and must see whichever agent is now
3070                // occupying the seat, not the one that just quota'd out —
3071                // otherwise it would judge session/continuation eligibility
3072                // by the wrong CLI and could resend a fallback's session id
3073                // to the agent that lost it the seat in the first place.
3074                job.spec = next.clone();
3075                let mut retry = job.clone();
3076                retry.seat = new_seat;
3077                retry.prompt = prompt::implement_in(
3078                    &instruction,
3079                    &job.cwd.to_string_lossy(),
3080                    &language,
3081                    &github_language,
3082                    brief.as_deref(),
3083                    &attachments,
3084                );
3085                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3086                let cache = self.state.config.cache_dir();
3087                let ctx = WaveCtx {
3088                    carry_seats: false,
3089                    pauses: Some((&self.pause, &self.interrupt)),
3090                    run: run_id,
3091                    node: "implement",
3092                    prompts,
3093                    cache: cache.as_deref(),
3094                    round: None,
3095                };
3096                let (fallback_seat, fallback_out) = run_one(
3097                    retry,
3098                    Arc::clone(&self.sem),
3099                    &ctx,
3100                    &mut self.state,
3101                    fallback_attempt,
3102                )
3103                .await;
3104                *seat = fallback_seat;
3105                *out = fallback_out;
3106            }
3107        }
3108        Ok(())
3109    }
3110
3111    /// Ask an implement seat's own CLI to confirm what it started, once, when
3112    /// its reply reported a command whose completion status it never
3113    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3114    /// that does and does not mean.
3115    ///
3116    /// The completion contract this task asks for, extended to `implement`
3117    /// with the same signal `continue_fix_report` reads for the fixer,
3118    /// rather than a keyword search over the reply or a hard requirement on
3119    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3120    /// a candidate's CLI turn ended cleanly while a test run it had started
3121    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3122    /// named in it at all is untouched by this: `commands` is empty, so
3123    /// there is nothing to be unconfirmed.
3124    ///
3125    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3126    /// this is not about recovering edits that might already be on disk, it
3127    /// is about a result the seat itself never vouched for, which resuming
3128    /// asks for regardless of what the tree already holds. Bounded to one
3129    /// attempt for the same reason `resume_undelivered` is — this is the
3130    /// most expensive node in the graph — and a seat that still cannot
3131    /// confirm on that attempt is left as whatever its (possibly still
3132    /// unconfirmed) reply says; this does not invent a new "failed" reason
3133    /// for a candidate that otherwise produced a real, committed change.
3134    async fn resume_unconfirmed_commands(
3135        &mut self,
3136        results: &mut [(usize, SeatState, AgentOutcome)],
3137        sent: &[SeatJob],
3138        prompts: &Prompts,
3139        run_id: &str,
3140    ) {
3141        for (wi, seat, out) in results.iter_mut() {
3142            let AgentOutcome::Ok(o) = &*out else {
3143                continue;
3144            };
3145            if !has_unconfirmed_command(&o.commands) {
3146                continue;
3147            }
3148            let Some(job) = sent.get(*wi) else { continue };
3149            if !has_context(&job.spec, seat, job.sessions) {
3150                self.state.event(
3151                    "implement",
3152                    format!(
3153                        "{}: the reply named a command whose own CLI never confirmed the exit \
3154                         status of, but there is no session left to resume",
3155                        seat.key
3156                    ),
3157                );
3158                continue;
3159            }
3160            self.state.event(
3161                "implement",
3162                format!(
3163                    "{}: the reply named a command whose own CLI never confirmed the exit \
3164                     status of; resuming the conversation",
3165                    seat.key
3166                ),
3167            );
3168            let mut retry = job.clone();
3169            retry.seat = seat.clone();
3170            retry.prompt = prompt::resume_incomplete(
3171                "a command in your last reply had no confirmed exit status",
3172            );
3173            retry.timeout = retry_budget(job.timeout, true);
3174            retry.stem = format!("{}-confirm", job.stem);
3175            let cache = self.state.config.cache_dir();
3176            let ctx = WaveCtx {
3177                carry_seats: false,
3178                pauses: Some((&self.pause, &self.interrupt)),
3179                run: run_id,
3180                node: "implement",
3181                prompts,
3182                cache: cache.as_deref(),
3183                round: None,
3184            };
3185            let (resumed_seat, resumed) =
3186                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3187            *seat = resumed_seat;
3188            *out = resumed;
3189        }
3190    }
3191
3192    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3193    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3194    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3195    /// that motivated this.
3196    ///
3197    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3198    /// own nudge loop already covers for judge/review/vote seats, and not a
3199    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3200    /// implement seats: here the CLI turn genuinely finished while the node's
3201    /// own work — the fixer's account of what it did — had not. Gated purely
3202    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3203    /// reply, never on any wording in it, so a fixer whose valid, first-try
3204    /// `FixReport` happens to mention having waited on a background test is
3205    /// never resumed — the `Ok(report)` branch at the call site returns
3206    /// before this is ever invoked.
3207    ///
3208    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3209    /// attempt ([`retry_budget`]), nothing attempted once the session is
3210    /// gone, and a quota hit ends the loop immediately rather than retrying a
3211    /// rate limit that fails the same way again.
3212    async fn continue_fix_report(
3213        &mut self,
3214        mut seat: SeatState,
3215        parse_err: String,
3216        job: &SeatJob,
3217        prompts: &Prompts,
3218        run_id: &str,
3219        round: usize,
3220    ) -> (
3221        SeatState,
3222        Option<FixReport>,
3223        Option<String>,
3224        ContinuationRecord,
3225    ) {
3226        let mut last_err = parse_err;
3227        let mut cumulative_wait_ms = 0u64;
3228        let mut attempts = 0usize;
3229        loop {
3230            if !has_context(&job.spec, &seat, job.sessions) {
3231                self.state.event(
3232                    "fix",
3233                    format!(
3234                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3235                         session left to resume into"
3236                    ),
3237                );
3238                let outcome = if attempts == 0 {
3239                    ContinuationOutcome::NoSession
3240                } else {
3241                    ContinuationOutcome::Exhausted
3242                };
3243                return (
3244                    seat,
3245                    None,
3246                    Some(format!("unparsable fix report: {last_err}")),
3247                    ContinuationRecord {
3248                        attempts,
3249                        cumulative_wait_ms,
3250                        outcome,
3251                    },
3252                );
3253            }
3254            if attempts >= MAX_FIX_CONTINUATIONS {
3255                self.state.event(
3256                    "fix",
3257                    format!(
3258                        "round {round}: fixer's reply still had no adoption report after \
3259                         {attempts} continuation(s) ({last_err}); giving up"
3260                    ),
3261                );
3262                return (
3263                    seat,
3264                    None,
3265                    Some(format!(
3266                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3267                    )),
3268                    ContinuationRecord {
3269                        attempts,
3270                        cumulative_wait_ms,
3271                        outcome: ContinuationOutcome::Exhausted,
3272                    },
3273                );
3274            }
3275            attempts += 1;
3276            self.state.event(
3277                "fix",
3278                format!(
3279                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3280                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3281                ),
3282            );
3283            let mut retry = job.clone();
3284            retry.seat = seat.clone();
3285            retry.prompt = prompt::resume_incomplete(&last_err);
3286            retry.timeout = retry_budget(job.timeout, true);
3287            retry.stem = format!("{}-continue{attempts}", job.stem);
3288            let cache = self.state.config.cache_dir();
3289            let ctx = WaveCtx {
3290                carry_seats: false,
3291                pauses: Some((&self.pause, &self.interrupt)),
3292                run: run_id,
3293                node: "fix",
3294                prompts,
3295                cache: cache.as_deref(),
3296                round: Some(round),
3297            };
3298            let (resumed_seat, resumed_out) = run_one(
3299                retry,
3300                Arc::clone(&self.sem),
3301                &ctx,
3302                &mut self.state,
3303                attempts,
3304            )
3305            .await;
3306            seat = resumed_seat;
3307            match resumed_out {
3308                AgentOutcome::Ok(o) => {
3309                    cumulative_wait_ms += o.duration_ms;
3310                    match verdict::extract_json::<FixReport>(&o.text) {
3311                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3312                            self.state.event(
3313                                "fix",
3314                                format!(
3315                                    "round {round}: fixer's adoption report recovered after \
3316                                     {attempts} continuation(s)"
3317                                ),
3318                            );
3319                            return (
3320                                seat,
3321                                Some(report),
3322                                None,
3323                                ContinuationRecord {
3324                                    attempts,
3325                                    cumulative_wait_ms,
3326                                    outcome: ContinuationOutcome::Resumed,
3327                                },
3328                            );
3329                        }
3330                        // The report parsed, but this same reply's own
3331                        // CommandEvidence — the identical record `state.jobs`
3332                        // renders — names a command whose CLI never
3333                        // confirmed an exit status. Read together, that is
3334                        // not a resolved answer: keep nudging rather than
3335                        // accept a report standing next to a command the
3336                        // seat's own CLI cannot vouch for.
3337                        Ok(_) => {
3338                            last_err = "the reply parsed, but it reported a command whose own CLI \
3339                                 never confirmed an exit status"
3340                                .to_owned();
3341                        }
3342                        Err(e) => last_err = e.to_string(),
3343                    }
3344                }
3345                AgentOutcome::Quota(o) => {
3346                    cumulative_wait_ms += o.duration_ms;
3347                    self.state.quota.push(QuotaLoss {
3348                        seat: seat.key.clone(),
3349                        node: "fix".to_owned(),
3350                        at: Timestamp::now(),
3351                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3352                    });
3353                    self.state.event(
3354                        "fix",
3355                        format!(
3356                            "round {round}: continuation rate limited (quota); not retrying now"
3357                        ),
3358                    );
3359                    return (
3360                        seat,
3361                        None,
3362                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3363                        ContinuationRecord {
3364                            attempts,
3365                            cumulative_wait_ms,
3366                            outcome: ContinuationOutcome::QuotaLost,
3367                        },
3368                    );
3369                }
3370                AgentOutcome::Dropped(o) => {
3371                    cumulative_wait_ms += o.duration_ms;
3372                    let why = o
3373                        .dropped
3374                        .as_ref()
3375                        .map(|d| d.why.as_str())
3376                        .unwrap_or("the CLI ended the stream without delivering its answer");
3377                    last_err = format!("the CLI dropped the stream ({why})");
3378                }
3379                AgentOutcome::Failed(e) => last_err = e,
3380                // The continuation was never started; the report stays lost
3381                // and the caller records that, without any handover.
3382                AgentOutcome::ResourceBlocked(e) => {
3383                    return (
3384                        seat,
3385                        None,
3386                        Some(format!("the continuation was not started: {e}")),
3387                        ContinuationRecord {
3388                            attempts: attempts.saturating_sub(1),
3389                            cumulative_wait_ms,
3390                            outcome: ContinuationOutcome::NoSession,
3391                        },
3392                    );
3393                }
3394            }
3395        }
3396    }
3397
3398    fn after_implement(&mut self) -> Result<()> {
3399        if !self.state.candidate_preservation.is_empty() {
3400            bail!("candidate preservation incomplete; judging is unsafe");
3401        }
3402        // Scan every candidate patch once the set is complete.
3403        if self.state.leaks.is_empty() {
3404            let cfg = self.state.config.blind.clone();
3405            let mut leaks = Vec::new();
3406            for c in &self.state.candidates {
3407                let Some(patch) =
3408                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3409                else {
3410                    continue;
3411                };
3412                leaks.extend(blind::scan(
3413                    &format!("candidate {} patch", c.label),
3414                    &patch,
3415                    &cfg.vendor_tokens,
3416                ));
3417            }
3418            if !leaks.is_empty() {
3419                let summary = leaks
3420                    .iter()
3421                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3422                    .collect::<Vec<_>>()
3423                    .join(", ");
3424                match cfg.on_leak {
3425                    LeakPolicy::Fail => {
3426                        self.state.status = RunStatus::Failed;
3427                        self.state
3428                            .event("blind", format!("vendor text in a patch: {summary}"));
3429                        self.state.leaks = leaks;
3430                        self.state.save()?;
3431                        self.settle_questions();
3432                        bail!(
3433                            "blind.on_leak = \"fail\" and vendor text reached a \
3434                             judged patch: {summary}"
3435                        );
3436                    }
3437                    LeakPolicy::Redact => self.state.event(
3438                        "blind",
3439                        format!("redacting vendor text for judging: {summary}"),
3440                    ),
3441                    LeakPolicy::Warn => self.state.event(
3442                        "blind",
3443                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3444                    ),
3445                }
3446                self.state.leaks = leaks;
3447            }
3448        }
3449
3450        if self.state.viable().is_empty() {
3451            if self.state.all_candidates_verified_noop() {
3452                // Every candidate agreed, with evidence the adoption guard
3453                // accepted, that nothing belongs in this worktree. That is
3454                // not the same fact as a candidate that simply failed to
3455                // write anything, and settling it as an ordinary `Failed`
3456                // (see `SCHEMA`'s doc for schema 10) is what let two of
3457                // task 391f's attempts burn a retry each re-discovering the
3458                // same already-landed fix. Terminal either way, so `judge`
3459                // must never run over an empty candidate set — unlike the
3460                // `Failed` branch below this returns `Ok`, not an error:
3461                // nothing here failed.
3462                self.state.status = RunStatus::VerifiedNoop;
3463                self.state.save()?;
3464                self.settle_questions();
3465                return Ok(());
3466            }
3467            self.state.status = RunStatus::Failed;
3468            self.state.save()?;
3469            self.settle_questions();
3470            bail!("no candidate produced a change; nothing to judge");
3471        }
3472        self.state.status = RunStatus::Judging;
3473        self.state.save()?;
3474        Ok(())
3475    }
3476
3477    // --------------------------------------------------------------- judge
3478
3479    async fn judge(&mut self) -> Result<()> {
3480        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3481        // agent files with `magi task add` name the run that paid for it. The
3482        // prompt overlay is cloned alongside it because the waves borrow it
3483        // while `self` is mutably borrowed by the node's own bookkeeping.
3484        let run_id = self.state.id.clone();
3485        let prompts = self.state.config.prompts.clone();
3486        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3487            return Ok(());
3488        }
3489        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3490        if viable.len() == 1 {
3491            // Recorded so this is a one-time event: `judgements` stays empty
3492            // either way, which without this flag is indistinguishable from
3493            // "not yet judged" on the next reentry — and status is left
3494            // untouched, so a later node's conclusion (e.g. `Blocked` after
3495            // the review budget ran out) survives a resume instead of being
3496            // clobbered back to `Judging` by this node running again.
3497            self.state.judge_skipped = true;
3498            self.state.event(
3499                "judge",
3500                format!(
3501                    "only candidate {} produced a change; judging skipped",
3502                    viable[0].label
3503                ),
3504            );
3505            self.state.save()?;
3506            return Ok(());
3507        }
3508        self.state.status = RunStatus::Judging;
3509
3510        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3511        let language = self.state.config.graph.language.clone();
3512        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3513        let sessions = self.state.config.graph.sessions;
3514        let artifacts = agent::artifacts_dir(&self.state.dir());
3515        let root = self.state.worktree_root();
3516        let base_short = short(&self.state.base_commit);
3517
3518        let mut jobs = Vec::new();
3519        let mut orders = Vec::new();
3520        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3521            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3522            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3523            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3524            let seat_key = format!("judge-{}", j + 1);
3525            let seat = self.seat(&seat_key, &spec.id);
3526            jobs.push(SeatJob {
3527                prompt: prompt::judge(
3528                    &self.state.instruction,
3529                    &views,
3530                    self.roles.judges.len(),
3531                    &base_short,
3532                    &language,
3533                ),
3534                spec,
3535                seat,
3536                cwd: root.join(format!("judge-{}", j + 1)),
3537                timeout,
3538                allow_write: false,
3539                sessions,
3540                artifacts: artifacts.clone(),
3541                stem: format!("judge-{}", j + 1),
3542                handover: None,
3543            });
3544        }
3545
3546        self.state.event(
3547            "judge",
3548            format!(
3549                "{} judges ranking {} candidates blind",
3550                jobs.len(),
3551                viable.len()
3552            ),
3553        );
3554        let labels_for_check = labels.clone();
3555        let mut quota_losses = Vec::new();
3556        let cache = self.state.config.cache_dir();
3557        let ctx = WaveCtx {
3558            carry_seats: false,
3559            pauses: Some((&self.pause, &self.interrupt)),
3560            run: &run_id,
3561            node: "judge",
3562            prompts: &prompts,
3563            cache: cache.as_deref(),
3564            round: None,
3565        };
3566        let results = ask_json_wave::<Ranking>(
3567            jobs,
3568            Arc::clone(&self.sem),
3569            self.state.config.graph.retries,
3570            &self.roles.judge_roster,
3571            &ctx,
3572            &mut quota_losses,
3573            &mut self.state,
3574            &move |r: &Ranking| r.validate(&labels_for_check),
3575        )
3576        .await;
3577        if self.state.resource_block.is_some() {
3578            return self.stop_for_resource();
3579        }
3580        self.state.quota.extend(quota_losses);
3581
3582        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3583            let agent_id = seat.agent.clone();
3584            self.state.seats.insert(seat.key.clone(), seat);
3585            let mut record = Judgement {
3586                judge: j + 1,
3587                seat: format!("judge-{}", j + 1),
3588                agent: agent_id,
3589                ranking: Vec::new(),
3590                reasons: BTreeMap::new(),
3591                confidence: None,
3592                strengths: BTreeMap::new(),
3593                order: orders[j].clone(),
3594                failed: None,
3595                duration_ms: 0,
3596            };
3597            match res {
3598                Ok((ranking, out)) => {
3599                    record.ranking = ranking.normalized();
3600                    record.reasons = ranking.reasons;
3601                    record.strengths = ranking.strengths;
3602                    record.confidence = ranking.confidence;
3603                    record.duration_ms = out.duration_ms;
3604                    self.state.event(
3605                        "judge",
3606                        format!(
3607                            "judge {} ranked {}",
3608                            j + 1,
3609                            record.ranking.iter().collect::<String>()
3610                        ),
3611                    );
3612                }
3613                Err(e) => {
3614                    record.failed = Some(e.to_string());
3615                    self.state
3616                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3617                }
3618            }
3619            self.state.judgements.push(record);
3620            self.state.save()?;
3621        }
3622        Ok(())
3623    }
3624
3625    // ---------------------------------------------------------- deliberate
3626
3627    async fn deliberate(&mut self) -> Result<()> {
3628        if self.hold_base_sync().await? {
3629            return Ok(());
3630        }
3631        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3632        // agent files with `magi task add` name the run that paid for it. The
3633        // prompt overlay is cloned alongside it because the waves borrow it
3634        // while `self` is mutably borrowed by the node's own bookkeeping.
3635        let run_id = self.state.id.clone();
3636        let prompts = self.state.config.prompts.clone();
3637        if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3638            return Ok(());
3639        }
3640        let tops: Vec<char> = self
3641            .state
3642            .judgements
3643            .iter()
3644            .filter_map(|j| j.ranking.first().copied())
3645            .collect();
3646        let rounds = self.state.config.graph.deliberate_rounds;
3647        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3648            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3649                self.state.event(
3650                    "deliberate",
3651                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3652                );
3653            }
3654            self.state.status = RunStatus::Voting;
3655            self.state.save()?;
3656            return Ok(());
3657        }
3658
3659        self.state.status = RunStatus::Deliberating;
3660        self.state.event(
3661            "deliberate",
3662            format!(
3663                "split: first choices were {} — opening {rounds} round(s)",
3664                tops.iter().collect::<String>()
3665            ),
3666        );
3667
3668        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3669        let language = self.state.config.graph.language.clone();
3670        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3671        let sessions = self.state.config.graph.sessions;
3672        let artifacts = agent::artifacts_dir(&self.state.dir());
3673        let root = self.state.worktree_root();
3674        let base_short = short(&self.state.base_commit);
3675
3676        // Judges argue in sequence so that a turn can answer the one before it;
3677        // that is the difference between deliberation and three parallel
3678        // monologues.
3679        for round in 1..=rounds {
3680            let mut turns: Vec<DeliberationTurn> = Vec::new();
3681            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3682                if self.state.judgements[j].failed.is_some() {
3683                    continue;
3684                }
3685                let seat_key = format!("judge-{}", j + 1);
3686                let spec = self.occupant(&seat_key, spec);
3687                let mut seat = self.seat(&seat_key, &spec.id);
3688                let transcript = self.transcript(&turns, j);
3689                let build = |context: Option<&str>| {
3690                    prompt::deliberate(
3691                        &self.state.instruction,
3692                        context,
3693                        &transcript,
3694                        round,
3695                        rounds,
3696                        &language,
3697                    )
3698                };
3699                let block = self.candidate_block(&viable, &base_short);
3700                let full = build(Some(&block));
3701                let text = if has_context(&spec, &seat, sessions) {
3702                    build(None)
3703                } else {
3704                    full.clone()
3705                };
3706                let job = SeatJob {
3707                    spec,
3708                    seat: seat.clone(),
3709                    prompt: text,
3710                    cwd: root.join(format!("judge-{}", j + 1)),
3711                    timeout,
3712                    allow_write: false,
3713                    sessions,
3714                    artifacts: artifacts.clone(),
3715                    stem: format!("delib-{round}-judge-{}", j + 1),
3716                    handover: Some(full),
3717                };
3718                let cache = self.state.config.cache_dir();
3719                let ctx = WaveCtx {
3720                    carry_seats: false,
3721                    pauses: Some((&self.pause, &self.interrupt)),
3722                    run: &run_id,
3723                    node: "deliberate",
3724                    prompts: &prompts,
3725                    cache: cache.as_deref(),
3726                    round: None,
3727                };
3728                // A turn is never nudged (`retries` 0): a failed seat is
3729                // handed to the next roster agent, which gets the full
3730                // context. An empty answer is a turn, not a failure.
3731                let mut losses = Vec::new();
3732                let mut results = ask_wave_with::<String>(
3733                    vec![job],
3734                    Arc::clone(&self.sem),
3735                    0,
3736                    &self.roles.judge_roster,
3737                    &ctx,
3738                    &mut losses,
3739                    &mut self.state,
3740                    &|text: &str| {
3741                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3742                    },
3743                )
3744                .await;
3745                self.state.quota.extend(losses);
3746                let (updated, res, _) = results.pop().expect("one job in, one result out");
3747                seat = updated;
3748                let agent_id = seat.agent.clone();
3749                self.state.seats.insert(seat.key.clone(), seat);
3750                let body = match res {
3751                    Ok((body, _)) => body,
3752                    // Skip the seat; a CLI's raw error JSON is never read as
3753                    // this judge's position.
3754                    Err(e) => {
3755                        self.state
3756                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3757                        continue;
3758                    }
3759                };
3760                let tentative = verdict::extract_json::<Position>(&body)
3761                    .ok()
3762                    .and_then(|p| p.tentative)
3763                    .and_then(|s| s.trim().chars().next())
3764                    .map(|c| c.to_ascii_uppercase());
3765                self.state.event(
3766                    "deliberate",
3767                    format!(
3768                        "round {round}: judge {} now favours {}",
3769                        j + 1,
3770                        tentative.map_or("—".to_owned(), |c| c.to_string())
3771                    ),
3772                );
3773                turns.push(DeliberationTurn {
3774                    judge: j + 1,
3775                    agent: agent_id,
3776                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3777                    tentative,
3778                });
3779            }
3780            self.state
3781                .deliberation
3782                .push(DeliberationRound { round, turns });
3783            self.state.save()?;
3784        }
3785
3786        self.state.status = RunStatus::Voting;
3787        self.state.save()?;
3788        Ok(())
3789    }
3790
3791    // ---------------------------------------------------------------- vote
3792
3793    async fn vote(&mut self) -> Result<()> {
3794        if self.hold_base_sync().await? {
3795            return Ok(());
3796        }
3797        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3798        // agent files with `magi task add` name the run that paid for it. The
3799        // prompt overlay is cloned alongside it because the waves borrow it
3800        // while `self` is mutably borrowed by the node's own bookkeeping.
3801        let run_id = self.state.id.clone();
3802        let prompts = self.state.config.prompts.clone();
3803        if !self.state.votes.is_empty() || self.state.tally.is_some() {
3804            return Ok(());
3805        }
3806        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3807        if viable.len() == 1 {
3808            return Ok(());
3809        }
3810        self.state.status = RunStatus::Voting;
3811
3812        let language = self.state.config.graph.language.clone();
3813        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3814        let sessions = self.state.config.graph.sessions;
3815        let artifacts = agent::artifacts_dir(&self.state.dir());
3816        let root = self.state.worktree_root();
3817        let base_short = short(&self.state.base_commit);
3818        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3819
3820        let mut jobs = Vec::new();
3821        let mut seats_at = Vec::new();
3822        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3823            if self
3824                .state
3825                .judgements
3826                .get(j)
3827                .is_some_and(|r| r.failed.is_some())
3828            {
3829                continue;
3830            }
3831            let seat_key = format!("judge-{}", j + 1);
3832            let spec = self.occupant(&seat_key, spec);
3833            let seat = self.seat(&seat_key, &spec.id);
3834            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3835            let text = if has_context(&spec, &seat, sessions) {
3836                prompt::final_vote(&viable, &language)
3837            } else {
3838                full.clone()
3839            };
3840            jobs.push(SeatJob {
3841                spec,
3842                seat,
3843                prompt: text,
3844                cwd: root.join(format!("judge-{}", j + 1)),
3845                timeout,
3846                allow_write: false,
3847                sessions,
3848                artifacts: artifacts.clone(),
3849                stem: format!("vote-judge-{}", j + 1),
3850                handover: Some(full),
3851            });
3852            seats_at.push(j);
3853        }
3854
3855        self.state.event(
3856            "vote",
3857            format!(
3858                "collecting {} final votes one by one, privately",
3859                jobs.len()
3860            ),
3861        );
3862        let allowed = viable.clone();
3863        let mut quota_losses = Vec::new();
3864        let cache = self.state.config.cache_dir();
3865        let ctx = WaveCtx {
3866            carry_seats: false,
3867            pauses: Some((&self.pause, &self.interrupt)),
3868            run: &run_id,
3869            node: "vote",
3870            prompts: &prompts,
3871            cache: cache.as_deref(),
3872            round: None,
3873        };
3874        let results = ask_json_wave::<FinalVote>(
3875            jobs,
3876            Arc::clone(&self.sem),
3877            self.state.config.graph.retries,
3878            &self.roles.judge_roster,
3879            &ctx,
3880            &mut quota_losses,
3881            &mut self.state,
3882            &move |v: &FinalVote| match v.label() {
3883                Some(c) if allowed.contains(&c) => Ok(()),
3884                other => bail!("vote {other:?} is not one of {allowed:?}"),
3885            },
3886        )
3887        .await;
3888        if self.state.resource_block.is_some() {
3889            return self.stop_for_resource();
3890        }
3891        self.state.quota.extend(quota_losses);
3892
3893        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3894            let agent_id = seat.agent.clone();
3895            self.state.seats.insert(seat.key.clone(), seat);
3896            let initial = self
3897                .state
3898                .judgements
3899                .get(j)
3900                .and_then(|r| r.ranking.first().copied());
3901            let mut record = VoteRecord {
3902                judge: j + 1,
3903                agent: agent_id,
3904                vote: None,
3905                reason: String::new(),
3906                changed: false,
3907            };
3908            match res {
3909                Ok((v, _)) => {
3910                    record.vote = v.label();
3911                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3912                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3913                    self.state.event(
3914                        "vote",
3915                        format!(
3916                            "judge {} voted {}{}",
3917                            j + 1,
3918                            record.vote.unwrap_or('?'),
3919                            if record.changed { " (changed)" } else { "" }
3920                        ),
3921                    );
3922                }
3923                Err(e) => {
3924                    self.state
3925                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3926                }
3927            }
3928            self.state.votes.push(record);
3929            self.state.save()?;
3930        }
3931        Ok(())
3932    }
3933
3934    // --------------------------------------------------------------- tally
3935
3936    fn tally(&mut self) -> Result<()> {
3937        if self.state.tally.is_some() {
3938            return Ok(());
3939        }
3940        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3941        let tops: Vec<char> = self
3942            .state
3943            .judgements
3944            .iter()
3945            .filter_map(|j| j.ranking.first().copied())
3946            .collect();
3947        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3948
3949        // A judge whose private vote failed still counted once, in the initial
3950        // ranking; using it beats discarding a whole seat.
3951        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3952        let mut cast: Vec<char> = Vec::new();
3953        for (i, j) in self.state.judgements.iter().enumerate() {
3954            let vote = self
3955                .state
3956                .votes
3957                .iter()
3958                .find(|v| v.judge == i + 1)
3959                .and_then(|v| v.vote)
3960                .or_else(|| j.ranking.first().copied());
3961            if let Some(v) = vote {
3962                *first_choice.entry(v).or_insert(0) += 1;
3963                cast.push(v);
3964            }
3965        }
3966
3967        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3968        for j in &self.state.judgements {
3969            let n = j.ranking.len();
3970            for (pos, label) in j.ranking.iter().enumerate() {
3971                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3972            }
3973        }
3974
3975        let best = first_choice.values().copied().max().unwrap_or(0);
3976        let mut leaders: Vec<char> = first_choice
3977            .iter()
3978            .filter(|(_, v)| **v == best)
3979            .map(|(k, _)| *k)
3980            .collect();
3981        let mut tie_break = None;
3982        if leaders.len() > 1 {
3983            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3984            let borda_leaders: Vec<char> = leaders
3985                .iter()
3986                .copied()
3987                .filter(|l| borda[l] == top_borda)
3988                .collect();
3989            tie_break = Some(if borda_leaders.len() == 1 {
3990                format!(
3991                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3992                    leaders.len()
3993                )
3994            } else {
3995                format!(
3996                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3997                    leaders.len()
3998                )
3999            });
4000            leaders = borda_leaders;
4001            leaders.sort_unstable();
4002        }
4003        let winner = *leaders
4004            .first()
4005            .or(viable.first())
4006            .context("no candidate to declare a winner from")?;
4007
4008        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4009        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4010        let deliberated = !self.state.deliberation.is_empty();
4011
4012        // Whose verdict is this? A rate-limited seat is absent even if it
4013        // ranked before the limit hit, so presence is measured against the
4014        // recorded losses, not just "did a ranking ever appear".
4015        let quota_seats: std::collections::BTreeSet<&str> =
4016            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4017        let mut present = 0usize;
4018        for (i, j) in self.state.judgements.iter().enumerate() {
4019            if quota_seats.contains(j.seat.as_str()) {
4020                continue;
4021            }
4022            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4023            let voted = self
4024                .state
4025                .votes
4026                .iter()
4027                .any(|v| v.judge == i + 1 && v.vote.is_some());
4028            if ranked || voted {
4029                present += 1;
4030            }
4031        }
4032        // Strict majority of the configured panel. A bare majority is real
4033        // signal we can act on, while a minority verdict must never stand in
4034        // for a healthy one. A one-candidate run needs no panel at all, and
4035        // `judges` stays `0` rather than the roster size a panel that never
4036        // sat would otherwise be credited with.
4037        let needs_quorum = viable.len() > 1;
4038        let judges_total = if needs_quorum {
4039            self.roles.judges.len()
4040        } else {
4041            0
4042        };
4043        let quorum = if needs_quorum {
4044            judges_total / 2 + 1
4045        } else {
4046            0
4047        };
4048        let met_quorum = !needs_quorum || present >= quorum;
4049        let uncontested = (!needs_quorum).then(|| {
4050            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4051        });
4052
4053        self.state.event(
4054            "tally",
4055            match &uncontested {
4056                Some(reason) => format!("winner {winner} — {reason}"),
4057                None => format!(
4058                    "winner {winner} — votes {} | initial {} | {} changed | \
4059                     {present}/{judges_total} judges{}",
4060                    first_choice
4061                        .iter()
4062                        .map(|(k, v)| format!("{k}:{v}"))
4063                        .collect::<Vec<_>>()
4064                        .join(" "),
4065                    if unanimous_initial {
4066                        "unanimous"
4067                    } else {
4068                        "split"
4069                    },
4070                    changed_votes,
4071                    if met_quorum {
4072                        String::new()
4073                    } else {
4074                        format!(" — below quorum ({quorum} required)")
4075                    },
4076                ),
4077            },
4078        );
4079        if !met_quorum {
4080            self.state.event(
4081                "stall",
4082                format!(
4083                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4084                     the run stops here, resumable"
4085                ),
4086            );
4087        }
4088        self.state.tally = Some(Tally {
4089            first_choice,
4090            borda,
4091            winner,
4092            rankings: tops.len(),
4093            unanimous_initial,
4094            deliberated,
4095            changed_votes,
4096            unanimous_final,
4097            tie_break,
4098            judges: judges_total,
4099            present,
4100            quorum,
4101            met_quorum,
4102            uncontested,
4103        });
4104        self.state.status = if met_quorum {
4105            RunStatus::Reviewing
4106        } else {
4107            RunStatus::Stalled
4108        };
4109        self.state.save()?;
4110        Ok(())
4111    }
4112
4113    // ------------------------------------------------------------- recover
4114
4115    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4116    /// resumed toward completion once the transient cause clears.
4117    ///
4118    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4119    /// it toward the quorum, which is exactly the set of seats whose absence
4120    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4121    /// not depend on which node happened to hit the limit), or an ordinary
4122    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4123    /// seat is never disturbed.
4124    ///
4125    /// A seat that now answers with a usable ranking is "recovered": its
4126    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4127    /// `tally` counts it present again), and its vote re-collected. A seat that
4128    /// still fails keeps its loss and stays absent.
4129    ///
4130    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4131    /// to review/gate/merge), `false` when it is still below quorum (the run
4132    /// stays `Stalled`, still resumable for a later retry).
4133    #[allow(clippy::too_many_lines)]
4134    async fn recover_stall(&mut self) -> Result<bool> {
4135        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4136        // agent files with `magi task add` name the run that paid for it. The
4137        // prompt overlay is cloned alongside it because the waves borrow it
4138        // while `self` is mutably borrowed by the node's own bookkeeping.
4139        let run_id = self.state.id.clone();
4140        let prompts = self.state.config.prompts.clone();
4141        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4142        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4143        // is what keeps a non-quota collapse — or a quota loss recorded at the
4144        // deliberate node — from being a permanent dead-end on `--resume`.
4145        let quota_seats: BTreeSet<&str> =
4146            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4147        let absent: Vec<String> = self
4148            .state
4149            .judgements
4150            .iter()
4151            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4152            .map(|j| j.seat.clone())
4153            .collect();
4154        if absent.is_empty() {
4155            return Ok(false);
4156        }
4157        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4158        if viable.len() <= 1 {
4159            return Ok(false);
4160        }
4161        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4162        let language = self.state.config.graph.language.clone();
4163        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4164        let sessions = self.state.config.graph.sessions;
4165        let artifacts = agent::artifacts_dir(&self.state.dir());
4166        let root = self.state.worktree_root();
4167        let base_short = short(&self.state.base_commit);
4168        let candidates: Vec<Candidate> = viable.clone();
4169
4170        // Map each absent seat key to its 0-based position in `roles.judges`.
4171        let mut positions: Vec<usize> = absent
4172            .iter()
4173            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4174            .collect();
4175        if positions.is_empty() {
4176            return Ok(false);
4177        }
4178        positions.sort_unstable();
4179        positions.dedup();
4180
4181        // Re-rank the lost seats, one blind prompt each.
4182        let mut judge_jobs = Vec::new();
4183        for &j in &positions {
4184            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4185            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4186            let seat_key = format!("judge-{}", j + 1);
4187            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4188            let seat = self.seat(&seat_key, &spec.id);
4189            judge_jobs.push(SeatJob {
4190                spec,
4191                seat,
4192                prompt: prompt::judge(
4193                    &self.state.instruction,
4194                    &views,
4195                    self.roles.judges.len(),
4196                    &base_short,
4197                    &language,
4198                ),
4199                cwd: root.join(seat_key),
4200                timeout,
4201                allow_write: false,
4202                sessions,
4203                artifacts: artifacts.clone(),
4204                stem: format!("judge-{}-recover", j + 1),
4205                handover: None,
4206            });
4207        }
4208
4209        let labels_for_check = labels.clone();
4210        let mut judge_losses = Vec::new();
4211        let retries = self.state.config.graph.retries;
4212        let cache = self.state.config.cache_dir();
4213        let ctx = WaveCtx {
4214            carry_seats: false,
4215            pauses: Some((&self.pause, &self.interrupt)),
4216            run: &run_id,
4217            node: "judge",
4218            prompts: &prompts,
4219            cache: cache.as_deref(),
4220            round: None,
4221        };
4222        let results = ask_json_wave::<Ranking>(
4223            judge_jobs,
4224            Arc::clone(&self.sem),
4225            retries,
4226            &self.roles.judge_roster,
4227            &ctx,
4228            &mut judge_losses,
4229            &mut self.state,
4230            &move |r: &Ranking| r.validate(&labels_for_check),
4231        )
4232        .await;
4233        if self.state.resource_block.is_some() {
4234            self.stop_for_resource()?;
4235            return Ok(false);
4236        }
4237
4238        // Refresh the judgement of every seat that ranked again.
4239        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4240        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4241            let agent_id = seat.agent.clone();
4242            self.state.seats.insert(seat.key.clone(), seat);
4243            let record = &mut self.state.judgements[j];
4244            match res {
4245                Ok((ranking, out)) => {
4246                    record.agent = agent_id;
4247                    record.ranking = ranking.normalized();
4248                    record.reasons = ranking.reasons;
4249                    record.strengths = ranking.strengths;
4250                    record.confidence = ranking.confidence;
4251                    record.failed = None;
4252                    record.duration_ms = out.duration_ms;
4253                    recovered.insert(j);
4254                    self.state.event(
4255                        "recover",
4256                        format!("judge {} ranked again after the limit", j + 1),
4257                    );
4258                }
4259                Err(e) => {
4260                    self.state
4261                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4262                }
4263            }
4264        }
4265
4266        // Re-ask the votes of the seats that recovered a ranking.
4267        let mut vote_jobs = Vec::new();
4268        let mut vote_pos: Vec<usize> = Vec::new();
4269        for &j in &recovered {
4270            let seat_key = format!("judge-{}", j + 1);
4271            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4272            let seat = self.seat(&seat_key, &spec.id);
4273            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4274            let text = if has_context(&spec, &seat, sessions) {
4275                prompt::final_vote(&labels, &language)
4276            } else {
4277                full.clone()
4278            };
4279            vote_jobs.push(SeatJob {
4280                spec,
4281                seat,
4282                prompt: text,
4283                cwd: root.join(seat_key),
4284                timeout,
4285                allow_write: false,
4286                sessions,
4287                artifacts: artifacts.clone(),
4288                stem: format!("vote-judge-{}-recover", j + 1),
4289                handover: Some(full),
4290            });
4291            vote_pos.push(j);
4292        }
4293        let allowed = labels.clone();
4294        let mut vote_losses = Vec::new();
4295        let vote_retries = self.state.config.graph.retries;
4296        let vote_cache = self.state.config.cache_dir();
4297        let ctx = WaveCtx {
4298            carry_seats: false,
4299            pauses: Some((&self.pause, &self.interrupt)),
4300            run: &run_id,
4301            node: "vote",
4302            prompts: &prompts,
4303            cache: vote_cache.as_deref(),
4304            round: None,
4305        };
4306        let votes = ask_json_wave::<FinalVote>(
4307            vote_jobs,
4308            Arc::clone(&self.sem),
4309            vote_retries,
4310            &self.roles.judge_roster,
4311            &ctx,
4312            &mut vote_losses,
4313            &mut self.state,
4314            &move |v: &FinalVote| match v.label() {
4315                Some(c) if allowed.contains(&c) => Ok(()),
4316                other => bail!("vote {other:?} is not one of {allowed:?}"),
4317            },
4318        )
4319        .await;
4320        if self.state.resource_block.is_some() {
4321            self.stop_for_resource()?;
4322            return Ok(false);
4323        }
4324        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4325            let agent_id = seat.agent.clone();
4326            self.state.seats.insert(seat.key.clone(), seat);
4327            match res {
4328                Ok((v, _)) => {
4329                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4330                        rec.vote = v.label();
4331                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4332                    } else {
4333                        self.state.votes.push(VoteRecord {
4334                            judge: j + 1,
4335                            agent: agent_id,
4336                            vote: v.label(),
4337                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4338                            changed: false,
4339                        });
4340                    }
4341                    self.state.event(
4342                        "recover",
4343                        format!("judge {} voted again after the limit", j + 1),
4344                    );
4345                }
4346                Err(e) => {
4347                    self.state
4348                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4349                }
4350            }
4351        }
4352
4353        // A seat that ranked again is present even if its re-vote failed —
4354        // `tally` falls back to the initial ranking's first choice — so clear
4355        // its quota loss. Seats that still fail keep theirs and stay absent.
4356        let recovered_keys: BTreeSet<String> = recovered
4357            .iter()
4358            .map(|&j| format!("judge-{}", j + 1))
4359            .collect();
4360        self.state
4361            .quota
4362            .retain(|q| !recovered_keys.contains(&q.seat));
4363        // A seat that hit the limit again is a fresh loss, not the old one:
4364        // replace the stale entry so the history stays one-per-seat and the
4365        // daemon can tell this attempt's loss from a previous session's.
4366        for loss in judge_losses.into_iter().chain(vote_losses) {
4367            if recovered_keys.contains(&loss.seat) {
4368                continue;
4369            }
4370            self.state.quota.retain(|q| q.seat != loss.seat);
4371            self.state.quota.push(loss);
4372        }
4373
4374        // Recompute the verdict from the refreshed panel.
4375        self.state.tally = None;
4376        self.tally()?;
4377        Ok(self
4378            .state
4379            .tally
4380            .as_ref()
4381            .map(|t| t.met_quorum)
4382            .unwrap_or(false))
4383    }
4384
4385    // ---------------------------------------------------------------- graft
4386
4387    /// Port the strengths the judges attributed to the losing candidates onto
4388    /// the winner, once, between the tally and the review loop.
4389    ///
4390    /// Best-effort by construction: it returns nothing and fails nothing. A
4391    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4392    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4393    /// the run carries on as if the phase were off. See [`graft`] for the
4394    /// decisions and `AGENTS.md` for the invariants.
4395    ///
4396    /// The one exception is a rollback that could not be confirmed: the
4397    /// rejected commit may still be the winner's tip, so the run is blocked
4398    /// (resumable; the record stays `Running`, which makes a resume retry the
4399    /// restore) instead of reviewing, gating and merging it.
4400    async fn graft(&mut self) -> Result<()> {
4401        let Err(e) = self.graft_inner().await else {
4402            return Ok(());
4403        };
4404        if e.is::<crate::repo_lock::ResourceBlocked>() {
4405            return Err(e);
4406        }
4407        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4408            self.state.status = RunStatus::Blocked;
4409            self.state.event("graft", format!("blocked: {e:#}"));
4410            self.state.save()?;
4411            return Err(e);
4412        }
4413        // An error after the `Running` record was saved may have left the
4414        // fixer's work in the tree: the record must not stay unsettled
4415        // (a later resume would reset past the review fixes), so confirm the
4416        // restore now, or block.
4417        if let Some(rec) = self.state.graft.clone()
4418            && !rec.outcome.settled()
4419        {
4420            if let Some(winner) = self.state.winner().cloned() {
4421                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4422                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4423                        return Err(re);
4424                    }
4425                    self.state.status = RunStatus::Blocked;
4426                    self.state.event("graft", format!("blocked: {re:#}"));
4427                    self.state.save()?;
4428                    return Err(re);
4429                }
4430            }
4431            if let Some(g) = self.state.graft.as_mut() {
4432                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4433            }
4434        }
4435        self.state.event("graft", format!("skipped: {e:#}"));
4436        if let Err(e) = self.state.save() {
4437            tracing::warn!("save after graft: {e:#}");
4438        }
4439        Ok(())
4440    }
4441
4442    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4443    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4444    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4445        let _coordination = self.coordinate(wt).await?;
4446        crate::recovery::restore_repair(wt, pre)
4447            .await
4448            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4449    }
4450
4451    /// A skip is a run event, recorded once however often the node is re-entered.
4452    fn graft_skipped(&mut self, why: &str) {
4453        let message = format!("skipped: {why}");
4454        if !self
4455            .state
4456            .events
4457            .iter()
4458            .any(|e| e.node == "graft" && e.message == message)
4459        {
4460            self.state.event("graft", message);
4461        }
4462    }
4463
4464    async fn graft_inner(&mut self) -> Result<()> {
4465        // A round that was started and never finished (crash, kill): the
4466        // fixer may have left anything in the tree, so put the winner back.
4467        if let Some(rec) = self.state.graft.clone()
4468            && !rec.outcome.settled()
4469        {
4470            // Reviews exist only if the graft ended without its record being
4471            // settled (a swallowed error): the tree has moved on since, and
4472            // a reset to `pre` would discard the review fixes.
4473            if self.state.reviews.is_empty()
4474                && let Some(winner) = self.state.winner().cloned()
4475            {
4476                self.graft_restore(&winner.worktree, &rec.pre).await?;
4477            }
4478            if rec.outcome == GraftOutcome::WaitingForRestore {
4479                self.state.graft = None;
4480                self.state.save()?;
4481            } else {
4482                if let Some(g) = self.state.graft.as_mut() {
4483                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4484                }
4485                self.state.event(
4486                    "graft",
4487                    "rolled back: the previous round was interrupted; the winner is untouched",
4488                );
4489                self.state.save()?;
4490                return Ok(());
4491            }
4492        }
4493        let plan = match graft::plan(&self.state) {
4494            Ok(plan) => plan,
4495            Err(skip) => {
4496                if skip != graft::Skip::AlreadyDone {
4497                    self.graft_skipped(skip.reason());
4498                    self.state.save()?;
4499                }
4500                return Ok(());
4501            }
4502        };
4503        let Some(winner) = self.state.winner().cloned() else {
4504            return Ok(());
4505        };
4506        let wt = winner.worktree.clone();
4507        if !git::is_clean(&wt).await? {
4508            self.graft_skipped("the winner's worktree has uncommitted changes");
4509            self.state.save()?;
4510            return Ok(());
4511        }
4512        let pre = git::rev_parse(&wt, "HEAD").await?;
4513        // Saved before the fixer is called: a stop from here on is repaired
4514        // by the branch above on the next entry.
4515        self.state.graft = Some(GraftRecord {
4516            pre: pre.clone(),
4517            post: None,
4518            lines: None,
4519            limit: None,
4520            agent: None,
4521            outcome: GraftOutcome::Running,
4522            requested: graft::requested(&plan),
4523            adoption: None,
4524            adoption_error: None,
4525        });
4526        self.state.event(
4527            "graft",
4528            format!(
4529                "porting strengths of candidate(s) {} onto {}",
4530                plan.strengths
4531                    .iter()
4532                    .map(|(l, _, _)| l.to_string())
4533                    .collect::<Vec<_>>()
4534                    .join(", "),
4535                plan.winner
4536            ),
4537        );
4538        self.state.save()?;
4539
4540        let fix_prompt = prompt::graft(
4541            &self.state.instruction,
4542            plan.winner,
4543            &plan.strengths,
4544            &self.state.config.graph.language,
4545        );
4546        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4547        let sessions = self.state.config.graph.sessions;
4548        let artifacts = agent::artifacts_dir(&self.state.dir());
4549        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4550        let (_, seat, out) = self
4551            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4552                SeatJob {
4553                    prompt: fix_prompt.clone(),
4554                    spec,
4555                    seat,
4556                    cwd: wt.clone(),
4557                    timeout,
4558                    allow_write: true,
4559                    sessions,
4560                    artifacts: artifacts.clone(),
4561                    stem: "graft".to_owned(),
4562                    handover: None,
4563                }
4564            })
4565            .await;
4566        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4567            // Preserve the need to restore earlier fallback edits without
4568            // confusing an unstarted seat with an interrupted invocation.
4569            self.state.graft.as_mut().expect("graft record").outcome =
4570                GraftOutcome::WaitingForRestore;
4571            return self.stop_for_resource();
4572        }
4573        let mut why: Option<String> = None;
4574        match &out {
4575            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4576                why = Some("the fixer produced nothing usable".to_owned());
4577            }
4578            AgentOutcome::Ok(_) => {}
4579            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4580            AgentOutcome::Quota(o) => {
4581                self.state.quota.push(QuotaLoss {
4582                    seat: seat.key.clone(),
4583                    node: "graft".to_owned(),
4584                    at: Timestamp::now(),
4585                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4586                });
4587                why = Some("rate limited (quota)".to_owned());
4588            }
4589            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4590                why = Some(format!(
4591                    "the fixer failed: {}",
4592                    e.lines()
4593                        .next()
4594                        .unwrap_or("")
4595                        .chars()
4596                        .take(160)
4597                        .collect::<String>()
4598                ));
4599            }
4600        }
4601        let agent_id = seat.agent.clone();
4602        self.state.seats.insert(seat.key.clone(), seat);
4603
4604        let mut post = None;
4605        let mut lines = None;
4606        let _coordination = self.coordinate(&wt).await?;
4607        if why.is_none() {
4608            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4609            if let Ok(r) = &rescued {
4610                self.state.note_withheld("graft", &r.withheld);
4611            }
4612            let head = git::rev_parse(&wt, "HEAD").await?;
4613            // What is measured must be what gets reviewed and gated: a failed
4614            // rescue, or anything still uncommitted (withheld lockfiles
4615            // aside, which are never part of the tree), rejects the graft.
4616            let dirty = match &rescued {
4617                Ok(r) => git::status_porcelain(&wt)
4618                    .await
4619                    .map(|st| {
4620                        st.lines().any(|l| {
4621                            !r.withheld
4622                                .iter()
4623                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4624                        })
4625                    })
4626                    .unwrap_or(true),
4627                Err(_) => true,
4628            };
4629            if dirty {
4630                why = Some("the fixer left changes that could not be committed".to_owned());
4631            } else if head == pre {
4632                why = Some("the fixer changed nothing".to_owned());
4633            } else {
4634                lines = git::changed_lines(&wt, &pre, &head).await;
4635                post = Some(head);
4636            }
4637        }
4638        let (adoption, adoption_error) = match &out {
4639            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4640            _ => (None, None),
4641        };
4642        let requested = graft::requested(&plan);
4643        let outcome = match why {
4644            None => {
4645                self.state.event(
4646                    "graft",
4647                    format!(
4648                        "applied: {}",
4649                        lines.map_or_else(
4650                            || "changed lines unavailable".to_owned(),
4651                            |n| format!("{n} changed lines")
4652                        )
4653                    ),
4654                );
4655                GraftOutcome::Applied
4656            }
4657            Some(why) => {
4658                // Not settled until the restore is confirmed: the record
4659                // stays `Running`, so a resume retries it.
4660                crate::recovery::restore_repair(&wt, &pre)
4661                    .await
4662                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4663                self.state.event(
4664                    "graft",
4665                    format!("rolled back: {why}; the winner is untouched"),
4666                );
4667                GraftOutcome::RolledBack(why)
4668            }
4669        };
4670        self.state.graft = Some(GraftRecord {
4671            pre,
4672            post,
4673            lines,
4674            limit: None,
4675            agent: Some(agent_id),
4676            outcome,
4677            requested,
4678            adoption,
4679            adoption_error,
4680        });
4681        self.state.save()?;
4682        Ok(())
4683    }
4684
4685    // ----------------------------------------------------------------- fold
4686
4687    async fn fold_losers(&mut self) -> Result<()> {
4688        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4689            return Ok(());
4690        };
4691        if !self.state.candidate_preservation.is_empty() {
4692            bail!("candidate preservation incomplete; refusing cleanup");
4693        }
4694        let repo = self.state.repo.clone();
4695        let _coordination = self.coordinate(&repo).await?;
4696        let mut folded = Vec::new();
4697        for i in 0..self.state.candidates.len() {
4698            let c = &self.state.candidates[i];
4699            if c.label == winner || c.folded {
4700                continue;
4701            }
4702            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4703            remove_preserved_worktree(&repo, &wt).await?;
4704            if git::branch_exists(&repo, &branch).await?
4705                && !git::branch_delete(&repo, &branch).await?
4706            {
4707                bail!("candidate branch removal refused");
4708            }
4709            self.state.candidates[i].folded = true;
4710            self.state.save()?;
4711            folded.push(label.to_string());
4712        }
4713        // The judges are finished; their checkouts are pure cost from here.
4714        let root = self.state.worktree_root();
4715        for j in 1..=self.roles.judges.len() {
4716            let wt = root.join(format!("judge-{j}"));
4717            if wt.exists() {
4718                remove_preserved_worktree(&repo, &wt).await?;
4719            }
4720        }
4721        // The design-deliberation stage is finished by the time a tally
4722        // exists — same reasoning as the judges above.
4723        if self.state.config.graph.advise {
4724            for k in 1..=self.state.config.graph.advisors {
4725                let wt = root.join(format!("advisor-{k}"));
4726                if wt.exists() {
4727                    remove_preserved_worktree(&repo, &wt).await?;
4728                }
4729            }
4730        }
4731        if !folded.is_empty() {
4732            self.state
4733                .event("fold", format!("folded candidates {}", folded.join(", ")));
4734            self.state.save()?;
4735        }
4736        Ok(())
4737    }
4738
4739    // ------------------------------------------------------------ base sync
4740
4741    async fn hold_base_sync(&mut self) -> Result<bool> {
4742        self.hold_base_sync_as(None).await
4743    }
4744
4745    async fn hold_base_sync_kind(
4746        &mut self,
4747        kind: crate::base_sync_recovery::StopKind,
4748    ) -> Result<()> {
4749        self.hold_base_sync_as(Some(kind)).await?;
4750        Ok(())
4751    }
4752
4753    async fn hold_base_sync_as(
4754        &mut self,
4755        kind: Option<crate::base_sync_recovery::StopKind>,
4756    ) -> Result<bool> {
4757        if !self.state.base_sync_blocked() {
4758            self.state.base_sync_recovery.stop = None;
4759            return Ok(false);
4760        }
4761        let head = match self.state.winner() {
4762            Some(w) => git::rev_parse(&w.worktree, "HEAD")
4763                .await
4764                .unwrap_or_default(),
4765            None => String::new(),
4766        };
4767        self.state.record_base_sync_stop(&head);
4768        if let Some(kind) = kind {
4769            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4770        }
4771        self.state.save()?;
4772        Ok(true)
4773    }
4774
4775    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4776    /// anything verifies it.
4777    ///
4778    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4779    /// read whatever is checked out in the winner's worktree. Left alone that
4780    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4781    /// it when the run *branched* - and a run takes long enough that the base
4782    /// has usually moved by the time it gets here. A gate that ran there
4783    /// answers "green on the commit this run started from", not "green on
4784    /// what is about to land", and the difference showed up three times in
4785    /// one day as a green run whose merge would have reverted a file another
4786    /// pull request had already landed.
4787    ///
4788    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4789    /// `land::Step::Rebase` calls, rather than a second implementation of the
4790    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4791    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4792    /// doc warns about ("two answers to a question nobody notices until a
4793    /// diff is wrong").
4794    ///
4795    /// A conflict is not the end of the road: the standing rebase is handed
4796    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4797    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4798    /// land). Once it finishes, review and the gate run as usual on the
4799    /// rebased tree, which is where a breakage the new base caused is caught
4800    /// by the ordinary gate-fix round. magi resolves nothing itself.
4801    ///
4802    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4803    /// `state.base_sync.attempts`, is how many times the base is *rebased
4804    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4805    /// *conflict* was given to a fixer. When the fixer cannot finish the
4806    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4807    /// what was tried (rounds spent, paths still conflicted) and the branch
4808    /// and worktree stay exactly as they were - untouched, for a person to
4809    /// look at - which is also what makes re-entering this function
4810    /// afterwards a no-op instead of a second attempt at the same wall. A
4811    /// push failure ends the same way.
4812    async fn sync_to_base(&mut self) -> Result<()> {
4813        if self.state.status == RunStatus::AlreadyInBase {
4814            return Ok(());
4815        }
4816        self.hold_base_sync().await?;
4817        let conflicted = self
4818            .state
4819            .base_sync
4820            .as_ref()
4821            .is_some_and(|s| s.conflict.is_some());
4822        let Some(winner) = self.state.winner().cloned() else {
4823            return Ok(());
4824        };
4825
4826        let repo = self.state.repo.clone();
4827        let remote = self.state.config.merge.remote.clone();
4828        let base_branch = self.state.base_branch.clone();
4829        let tracking = format!("{remote}/{base_branch}");
4830
4831        let fetched_base =
4832            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4833        // No network, or the remote never had this branch: the run already
4834        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4835        // otherwise), and one that got this far is not blocked by a fetch
4836        // that fails now. It just has no newer tip to compare against.
4837        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4838            return Ok(());
4839        };
4840
4841        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4842        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4843        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4844
4845        // Before any rebase, and before a recorded conflict is honoured: a
4846        // branch whose change reached the base under other commit ids has
4847        // nothing to rebase and nothing to conflict with, and a run that
4848        // already stopped on that phantom conflict recovers here on resume.
4849        // `behind == 0` with head == tip is a branch the base has since taken
4850        // in whole, whether or not a conflict was ever recorded: the ancestry
4851        // proof must still run (`classify` ignores a head still on the start
4852        // commit).
4853        if (behind > 0 || conflicted || head == tip)
4854            && self
4855                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4856                .await?
4857        {
4858            return Ok(());
4859        }
4860        if conflicted && self.state.base_recovery_pending() {
4861            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4862            let owner =
4863                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4864            if !fetched_base
4865                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4866                || !git::is_clean(&winner.worktree).await?
4867                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4868            {
4869                return Ok(());
4870            }
4871        }
4872        let recovery = conflicted && self.state.consume_base_recovery(&head);
4873        if conflicted {
4874            // Consumption (including a refused changed HEAD) is durable before Git.
4875            self.state.save()?;
4876            if !recovery {
4877                return Ok(());
4878            }
4879        }
4880
4881        if behind == 0 {
4882            // A fixer-finished rebase moves the branch ref before the
4883            // winner's worktree is told (`sync_to_head` below). A run that
4884            // died in between resumes here with `behind == 0` and a tree still
4885            // holding the pre-rebase files, which review and the gate would
4886            // then read. That state is exactly: HEAD moved off the tip the
4887            // rebase started from, yet the tree is still identical to that
4888            // tip. A tree with edits of its own differs from it, so nothing
4889            // is thrown away.
4890            if let Some(from) = self
4891                .state
4892                .rebase_fixes
4893                .iter()
4894                .rev()
4895                .find_map(|r| r.from.clone())
4896                && from != head
4897                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4898                    .await
4899                    .is_ok_and(|o| o.ok())
4900            {
4901                git::sync_to_head(&winner.worktree).await?;
4902            }
4903            // An earlier attempt may have rebased the branch locally and died
4904            // before pushing it (only the fresh-rebase arm below pushes).
4905            // Publish it now, so the plain push at PR time is not refused as
4906            // a non-fast-forward. A run already holding a recorded conflict
4907            // never reaches here; that case is out of scope.
4908            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4909            if let Some(why) = &conflict {
4910                self.state.status = RunStatus::Blocked;
4911                self.state.event("land", why.clone());
4912            }
4913            self.state.base_sync = Some(BaseSync {
4914                tip,
4915                behind: 0,
4916                attempts: attempts + usize::from(recovery),
4917                conflict,
4918                already_in: None,
4919            });
4920            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4921                .await?;
4922            self.state.save()?;
4923            return Ok(());
4924        }
4925
4926        if attempts >= BASE_SYNC_ROUNDS && !recovery {
4927            let why = format!(
4928                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4929                 rebase(s); rebasing again would only race it",
4930                winner.branch
4931            );
4932            self.state.status = RunStatus::Blocked;
4933            self.state.base_sync = Some(BaseSync {
4934                tip,
4935                behind,
4936                attempts,
4937                conflict: Some(why.clone()),
4938                already_in: None,
4939            });
4940            self.state.event("land", why);
4941            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4942                .await?;
4943            self.state.save()?;
4944            return Ok(());
4945        }
4946
4947        self.state.event(
4948            "land",
4949            format!(
4950                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4951                winner.branch
4952            ),
4953        );
4954        self.state.save()?;
4955
4956        // The remote's copy of the branch, read now and only if the fetch
4957        // really succeeded (a stale tracking ref must never pin a lease). It is
4958        // pushed over after a rebase only when it is a commit this branch
4959        // already contains, by ancestry or by patch (an earlier rebase of ours
4960        // that never reached the remote): anything else is somebody else's work.
4961        let branch_tracking = format!("{remote}/{}", winner.branch);
4962        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4963        let remote_tip = if self.state.task_pr_branch().is_none()
4964            && matches!(&fetched_branch, Ok(o) if o.ok())
4965        {
4966            git::rev_parse(&repo, &branch_tracking).await.ok()
4967        } else {
4968            None
4969        };
4970        // A remote tip this branch does not contain is somebody else's work:
4971        // rebasing would leave a local tip that can never be pushed. Stop
4972        // before touching anything and say so.
4973        if let Some(theirs) = &remote_tip
4974            && !git::is_ancestor(&repo, theirs, &head).await
4975            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4976                .await
4977                .is_ok_and(|missing| missing.is_empty())
4978        {
4979            let why = format!(
4980                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4981                 them",
4982                short(theirs),
4983                winner.branch
4984            );
4985            self.state.status = RunStatus::Blocked;
4986            self.state.base_sync = Some(BaseSync {
4987                tip,
4988                behind,
4989                attempts: attempts + usize::from(recovery),
4990                conflict: Some(why.clone()),
4991                already_in: None,
4992            });
4993            self.state.event("land", why);
4994            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4995                .await?;
4996            self.state.save()?;
4997            return Ok(());
4998        }
4999
5000        let scratch = self.state.dir().join("base-sync");
5001        let rebased =
5002            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5003                .await
5004            {
5005                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5006                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5007                Err(e) => Err(e),
5008            };
5009        let attempts = attempts + 1;
5010        let stop_kind = match &rebased {
5011            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5012            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5013            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5014        };
5015        match rebased {
5016            Ok(None) => {
5017                // The branch ref moved, but a worktree that already had it
5018                // checked out (the winner's) was not told; sync its index and
5019                // files before anything reads them.
5020                git::sync_to_head(&winner.worktree).await?;
5021                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5022                let mut conflict = None;
5023                if let Some(pinned) = &remote_tip {
5024                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5025                    match pushed {
5026                        Ok(o) if o.ok() => self.state.event(
5027                            "land",
5028                            format!("pushed rebased {} to {remote}", winner.branch),
5029                        ),
5030                        Ok(o) => {
5031                            conflict = Some(format!(
5032                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5033                                winner.branch,
5034                                short(pinned),
5035                                o.stderr.chars().take(600).collect::<String>()
5036                            ));
5037                        }
5038                        Err(e) => {
5039                            conflict = Some(format!(
5040                                "rebased {} locally but could not push it: {e:#}",
5041                                winner.branch
5042                            ));
5043                        }
5044                    }
5045                }
5046                if let Some(why) = &conflict {
5047                    self.state.status = RunStatus::Blocked;
5048                    self.state.event("land", why.clone());
5049                }
5050                self.state.base_sync = Some(BaseSync {
5051                    tip: tip.clone(),
5052                    behind: 0,
5053                    attempts,
5054                    conflict,
5055                    already_in: None,
5056                });
5057                self.state
5058                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5059            }
5060            Ok(Some(conflict)) => {
5061                let why = format!(
5062                    "{} conflicts with {tracking} and did not rebase: {}",
5063                    winner.branch,
5064                    conflict.chars().take(600).collect::<String>()
5065                );
5066                self.state.status = RunStatus::Blocked;
5067                self.state.base_sync = Some(BaseSync {
5068                    tip,
5069                    behind,
5070                    attempts,
5071                    conflict: Some(why.clone()),
5072                    already_in: None,
5073                });
5074                self.state.event("land", why);
5075            }
5076            // A refused Git write or a busy repository: nothing is wrong with
5077            // the branch. The scratch rebase stays standing and the run
5078            // resumes into it (`base_sync` must not claim a conflict).
5079            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5080                self.state.save()?;
5081                return Ok(());
5082            }
5083            Err(e) => {
5084                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5085                self.state.status = RunStatus::Blocked;
5086                self.state.base_sync = Some(BaseSync {
5087                    tip,
5088                    behind,
5089                    attempts,
5090                    conflict: Some(why.clone()),
5091                    already_in: None,
5092                });
5093                self.state.event("land", why);
5094            }
5095        }
5096        if !self.state.base_sync_blocked() {
5097            self.state.base_sync_recovery.stop = None;
5098        }
5099        self.hold_base_sync_kind(stop_kind).await?;
5100        self.state.save()?;
5101        Ok(())
5102    }
5103
5104    /// Push a branch an earlier attempt rebased locally but never published,
5105    /// pinned to the remote tip read right after a successful fetch. Returns
5106    /// the reason when the run must stop; `None` when there was nothing to do
5107    /// (no remote copy, the same tip, or a remote copy this branch already
5108    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5109    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5110        // Task competitions publish only after the PR text gate, to their
5111        // public head; never resurrect a candidate branch on the remote.
5112        if self.state.task_pr_branch().is_some() {
5113            return None;
5114        }
5115        let repo = self.state.repo.clone();
5116        let remote = self.state.config.merge.remote.clone();
5117        let fetched = git::fetch(&repo, &remote, branch).await;
5118        if !matches!(&fetched, Ok(o) if o.ok()) {
5119            return None;
5120        }
5121        let branch_tracking = format!("{remote}/{branch}");
5122        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5123        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5124            return None;
5125        }
5126        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5127            .await
5128            .is_ok_and(|missing| missing.is_empty())
5129        {
5130            return Some(format!(
5131                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5132                 them",
5133                short(&theirs)
5134            ));
5135        }
5136        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5137            Ok(o) if o.ok() => {
5138                self.state
5139                    .event("land", format!("pushed rebased {branch} to {remote}"));
5140                None
5141            }
5142            Ok(o) => Some(format!(
5143                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5144                 someone may have pushed): {}",
5145                short(&theirs),
5146                o.stderr.chars().take(600).collect::<String>()
5147            )),
5148            Err(e) => Some(format!(
5149                "{branch} is rebased locally but could not be pushed: {e:#}"
5150            )),
5151        }
5152    }
5153
5154    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5155    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5156    /// whether it did.
5157    ///
5158    /// Checked only when the base is ahead of the branch. A failing check is
5159    /// "not proven" - the ordinary rebase path then decides - never a reason to
5160    /// stop the run.
5161    ///
5162    /// The remote copy of the branch is held to the same standard as the local
5163    /// one: if it carries a tip this worktree does not, that tip must itself be
5164    /// proven in the base, or nothing is settled (a pull request would
5165    /// otherwise be closed over commits nobody checked). The pull request is
5166    /// closed *before* the terminal status is saved; if that fails for a
5167    /// reason other than a refusal (no network, a `gh` error) the run is left
5168    /// `Blocked` with the reason as its conflict, which a resume retries -
5169    /// the same recovery a phantom conflict gets.
5170    async fn settle_already_in(
5171        &mut self,
5172        branch: &str,
5173        tip: &str,
5174        head: &str,
5175        attempts: usize,
5176        behind: usize,
5177    ) -> Result<bool> {
5178        let repo = self.state.repo.clone();
5179        let remote = self.state.config.merge.remote.clone();
5180        let start = self.state.base_commit.clone();
5181        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5182            Ok(Some(e)) => e,
5183            Ok(None) => return Ok(false),
5184            Err(e) => {
5185                tracing::warn!("already-in-base check for {branch}: {e:#}");
5186                return Ok(false);
5187            }
5188        };
5189        let mut verified = vec![head.to_owned()];
5190        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5191        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5192        if matches!(&fetched, Ok(o) if o.ok())
5193            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5194            && theirs != head
5195        {
5196            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5197                Ok(Some(_)) => verified.push(theirs),
5198                _ => return Ok(false),
5199            }
5200        }
5201        let base_branch = self.state.base_branch.clone();
5202        let message = format!(
5203            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5204             land",
5205            evidence.names(),
5206            evidence.proof.as_str()
5207        );
5208        let closed =
5209            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5210                .await;
5211        match closed {
5212            Ok(Ok(url)) => self
5213                .state
5214                .event("land", format!("closed {url}: superseded on {base_branch}")),
5215            Ok(Err(why)) => self
5216                .state
5217                .event("land", format!("did not close a pull request: {why}")),
5218            Err(e) => {
5219                let why = format!(
5220                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5221                     not be closed ({e:#}); resume to retry"
5222                );
5223                self.state.status = RunStatus::Blocked;
5224                self.state.base_sync = Some(BaseSync {
5225                    tip: tip.to_owned(),
5226                    behind,
5227                    attempts,
5228                    conflict: Some(why.clone()),
5229                    already_in: None,
5230                });
5231                self.state.event("land", why);
5232                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5233                    .await?;
5234                self.state.save()?;
5235                return Ok(true);
5236            }
5237        }
5238        self.state.status = RunStatus::AlreadyInBase;
5239        self.state.base_sync_recovery.stop = None;
5240        self.state.base_sync = Some(BaseSync {
5241            tip: tip.to_owned(),
5242            behind,
5243            attempts,
5244            conflict: None,
5245            already_in: Some(evidence),
5246        });
5247        self.state.event("land", message);
5248        self.state.save()?;
5249        self.settle_questions();
5250        Ok(true)
5251    }
5252
5253    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5254    /// last landed the winner on, once it has run, else the commit the run
5255    /// branched from.
5256    ///
5257    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5258    /// and `vote` all happen before there is a winner to rebase, so they
5259    /// compare every candidate against the branch point on purpose, and a
5260    /// base that moves after they are already done cannot change an answer
5261    /// they already gave.
5262    fn landing_base(&self) -> String {
5263        self.state
5264            .base_sync
5265            .as_ref()
5266            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5267    }
5268
5269    // ------------------------------------------------------- operator fix
5270
5271    /// Route specific, already-recorded review findings to a fixer for a
5272    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5273    /// entry point.
5274    ///
5275    /// Distinct from `review_loop`'s own fix step in three ways: it never
5276    /// runs a reviewer wave, it never spends review-round budget, and what
5277    /// happened is recorded as an [`OperatorFixRequest`] appended to
5278    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5279    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5280    /// and vote must never be rewritten to look like a manufactured blocking
5281    /// verdict.
5282    ///
5283    /// Only meaningful once review has actually concluded: `Ready` (handed
5284    /// off with findings still open, or simply concluded clean while minor
5285    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5286    /// gate failed). Everything else is refused: a run still in progress
5287    /// should simply be resumed, and a `Merged` run's branch has already
5288    /// landed — reopening *this* run's own record cannot change that, so the
5289    /// answer there is a fresh `magi review <branch>`.
5290    ///
5291    /// A real commit here re-verifies through a fresh, ordinary review-only
5292    /// run on the same branch ([`Self::review`]) rather than reopening this
5293    /// run's own `review_loop`: once any round in this run's history went
5294    /// clean, `review_conclusion` treats that as permanent by design (the
5295    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5296    /// way to force one more genuine reviewer wave out of *this* run without
5297    /// either rewriting history or weakening that guarantee for every other
5298    /// caller. A review-only run costs nothing extra — no implementation, no
5299    /// judging, no vote — and exercises the exact same review → verify →
5300    /// gate → (human) merge path, unmodified.
5301    pub async fn fix_selected(
5302        &mut self,
5303        ids: &[String],
5304        reason: &str,
5305        allow_stale: bool,
5306    ) -> Result<()> {
5307        let reason = reason.trim();
5308        if reason.is_empty() {
5309            bail!("a fix request needs a reason — that is the operator's own record of why");
5310        }
5311        if ids.is_empty() {
5312            bail!("no finding id given");
5313        }
5314        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5315            bail!(
5316                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5317                 has already concluded — can be given a targeted fix. A run still \
5318                 in progress should simply be resumed; a `merged` run's branch has \
5319                 already landed, so its answer is a fresh `magi review <branch>`, \
5320                 not reopening this run's own record",
5321                self.state.id,
5322                self.state.status.as_str()
5323            );
5324        }
5325        let Some(winner) = self.state.winner().cloned() else {
5326            bail!("run {} has no winning candidate to fix", self.state.id);
5327        };
5328        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5329            bail!(
5330                "branch `{}` no longer exists; this run cannot be extended",
5331                winner.branch
5332            );
5333        }
5334        let home = crate::run::home();
5335        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5336            bail!(
5337                "run {} is currently being worked on by another magi process",
5338                self.state.id
5339            );
5340        }
5341        // Held for the rest of this call, including the follow-up review
5342        // below: two `magi fix` invocations against the same run must not
5343        // both reach the worktree manipulation further down, which would
5344        // otherwise race to remove and recreate the same directory — see
5345        // [`FixClaim`]'s own doc.
5346        let _claim = FixClaim::acquire(&self.state.dir())?;
5347
5348        // Resolve every id before spending anything — an unknown id refuses
5349        // the whole request rather than silently dropping it — and dedup
5350        // while keeping the operator's own order.
5351        let mut seen = BTreeSet::new();
5352        let mut findings = Vec::new();
5353        let mut missing = Vec::new();
5354        for id in ids {
5355            if !seen.insert(id.clone()) {
5356                continue;
5357            }
5358            match self.state.finding(id) {
5359                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5360                    id: f.id.clone(),
5361                    severity: f.severity,
5362                    reviewer_vote: rec.vote,
5363                    round: round.round,
5364                    round_head: round.head.clone(),
5365                    reviewer: rec.reviewer,
5366                    agent: rec.agent.clone(),
5367                    file: f.file.clone(),
5368                    line: f.line,
5369                    title: f.title.clone(),
5370                    detail: f.detail.clone(),
5371                    outcome: OperatorFixOutcome::Pending,
5372                }),
5373                None => missing.push(id.clone()),
5374            }
5375        }
5376        if !missing.is_empty() {
5377            bail!(
5378                "unknown finding id(s): {}; nothing was changed",
5379                missing.join(", ")
5380            );
5381        }
5382
5383        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5384        let stale_details: Vec<(String, String)> = findings
5385            .iter()
5386            .filter(|f| f.round_head != head_at_request)
5387            .map(|f| (f.id.clone(), f.round_head.clone()))
5388            .collect();
5389        let stale = !stale_details.is_empty();
5390        if stale && !allow_stale {
5391            bail!(
5392                "the branch has moved since some finding(s) were raised — {} — now \
5393                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5394                stale_details
5395                    .iter()
5396                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5397                    .collect::<Vec<_>>()
5398                    .join(", "),
5399                short(&head_at_request)
5400            );
5401        }
5402
5403        let request = OperatorFixRequest {
5404            requested_at: Timestamp::now(),
5405            reason: reason.to_owned(),
5406            findings,
5407            head_at_request: head_at_request.clone(),
5408            allow_stale,
5409            stale,
5410            fix: None,
5411            result_head: None,
5412            follow_up_review_run: None,
5413        };
5414        self.state.event(
5415            "fix",
5416            format!(
5417                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5418                request.findings.len(),
5419                request
5420                    .findings
5421                    .iter()
5422                    .map(|f| f.id.as_str())
5423                    .collect::<Vec<_>>()
5424                    .join(", "),
5425            ),
5426        );
5427        // Recorded now, before any worktree work or the fixer call itself —
5428        // and re-saved at each checkpoint below: a crash at any point after
5429        // this (mid fixer call, mid follow-up review) must not lose the fact
5430        // that this was requested, for which findings, and why. Everything
5431        // past this point reads and writes through `request_index` rather
5432        // than a local variable, since `request` itself is moved here.
5433        self.state.operator_fixes.push(request);
5434        self.state.save()?;
5435        let request_index = self.state.operator_fixes.len() - 1;
5436
5437        // A fresh, dedicated worktree for this one call, never the winner's
5438        // own worktree in place: that one may already be gone (folded away),
5439        // and reusing it in place would leave the branch checked out there
5440        // when the follow-up review below tries to check it out again. Freed
5441        // immediately after, either way — but only once confirmed clean:
5442        // `worktree_remove` is a `git worktree remove --force`, which would
5443        // otherwise discard uncommitted work left there by the operator or
5444        // another process before this had a chance to even look at it.
5445        let repo = self.state.repo.clone();
5446        let coordination = self.coordinate(&repo).await?;
5447        if winner.worktree.exists() {
5448            // Lockfiles a rescue commit withheld stay untracked on purpose and
5449            // are already recorded; they are not the operator's work to protect.
5450            let dirty = git::git(
5451                &winner.worktree,
5452                &["status", "--porcelain", "--untracked-files=all"],
5453            )
5454            .await?;
5455            let only_withheld = dirty.lines().all(|l| {
5456                l.strip_prefix("?? ")
5457                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5458            });
5459            if !only_withheld {
5460                bail!(
5461                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5462                     discard them first",
5463                    winner.worktree.display()
5464                );
5465            }
5466            git::worktree_remove(&self.state.repo, &winner.worktree)
5467                .await
5468                .ok();
5469        }
5470        let fix_worktree = self.state.worktree_root().join("operator-fix");
5471        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5472        git::git(
5473            &self.state.repo,
5474            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5475        )
5476        .await
5477        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5478        if !git::is_clean(&fix_worktree).await? {
5479            git::worktree_remove(&self.state.repo, &fix_worktree)
5480                .await
5481                .ok();
5482            bail!(
5483                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5484                winner.branch
5485            );
5486        }
5487        let repo_for_setup = self.state.repo.clone();
5488        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5489            Ok(()) => {
5490                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5491            }
5492            Err(e) => Err(e),
5493        };
5494        if let Err(e) = setup {
5495            git::worktree_remove(&self.state.repo, &fix_worktree)
5496                .await
5497                .ok();
5498            return Err(e);
5499        }
5500
5501        let run_id = self.state.id.clone();
5502        let prompts = self.state.config.prompts.clone();
5503        let language = self.state.config.graph.language.clone();
5504        let sessions = self.state.config.graph.sessions;
5505        let artifacts = agent::artifacts_dir(&self.state.dir());
5506        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5507            .findings
5508            .iter()
5509            .map(|f| Finding {
5510                id: f.id.clone(),
5511                severity: f.severity,
5512                file: f.file.clone(),
5513                line: f.line,
5514                title: f.title.clone(),
5515                detail: f.detail.clone(),
5516            })
5517            .collect();
5518        let fix_prompt = prompt::operator_fix_in(
5519            &self.state.instruction,
5520            &finding_list,
5521            reason,
5522            &stale_details,
5523            &head_at_request,
5524            &language,
5525            &self.state.config.graph.github_language,
5526        );
5527        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5528        drop(coordination);
5529        let (job, seat, out) = self
5530            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5531                prompt: fix_prompt.clone(),
5532                spec,
5533                seat,
5534                cwd: fix_worktree.clone(),
5535                timeout,
5536                allow_write: true,
5537                sessions,
5538                artifacts: artifacts.clone(),
5539                stem: "operator-fix".to_owned(),
5540                handover: None,
5541            })
5542            .await;
5543        let agent_id = seat.agent.clone();
5544
5545        let mut fix = FixRecord {
5546            agent: agent_id,
5547            addressed: Vec::new(),
5548            rejected: Vec::new(),
5549            notes: String::new(),
5550            committed: false,
5551            failed: None,
5552            duration_ms: 0,
5553            continuation: None,
5554        };
5555        let mut final_seat = seat.clone();
5556        match out {
5557            AgentOutcome::Ok(o) => {
5558                fix.duration_ms = o.duration_ms;
5559                let parsed = verdict::extract_json::<FixReport>(&o.text);
5560                let incomplete_reason = match &parsed {
5561                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5562                        "the reply parsed, but it reported a command whose own CLI \
5563                         never confirmed an exit status"
5564                            .to_owned(),
5565                    ),
5566                    Ok(_) => None,
5567                    Err(e) => Some(e.to_string()),
5568                };
5569                match incomplete_reason {
5570                    None => {
5571                        let report = parsed.expect("checked Ok above");
5572                        fix.addressed = report.addressed;
5573                        fix.rejected = report.rejected;
5574                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5575                    }
5576                    Some(reason) => {
5577                        let (resumed_seat, resolved, failure, cont) = self
5578                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5579                            .await;
5580                        fix.duration_ms += cont.cumulative_wait_ms;
5581                        fix.continuation = Some(cont);
5582                        final_seat = resumed_seat;
5583                        match resolved {
5584                            Some(report) => {
5585                                fix.addressed = report.addressed;
5586                                fix.rejected = report.rejected;
5587                                fix.notes =
5588                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5589                            }
5590                            None => fix.failed = failure,
5591                        }
5592                    }
5593                }
5594            }
5595            AgentOutcome::Dropped(o) => {
5596                fix.duration_ms = o.duration_ms;
5597                let why = o
5598                    .dropped
5599                    .as_ref()
5600                    .map(|d| d.why.as_str())
5601                    .unwrap_or("the CLI ended the stream without delivering its answer");
5602                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5603            }
5604            AgentOutcome::Quota(o) => {
5605                self.state.quota.push(QuotaLoss {
5606                    seat: final_seat.key.clone(),
5607                    node: "fix".to_owned(),
5608                    at: Timestamp::now(),
5609                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5610                });
5611                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5612            }
5613            AgentOutcome::Failed(e) => fix.failed = Some(e),
5614            AgentOutcome::ResourceBlocked(e) => {
5615                fix.failed = Some(format!("the fixer was not started: {e}"));
5616            }
5617        }
5618        if fix.continuation.is_none() {
5619            fix.continuation = Some(ContinuationRecord::not_needed());
5620        }
5621        self.state.seats.insert(final_seat.key.clone(), final_seat);
5622
5623        let rescue_message = format!(
5624            "magi: operator-selected fix ({}) (uncommitted work)",
5625            self.state.operator_fixes[request_index]
5626                .findings
5627                .iter()
5628                .map(|f| f.id.as_str())
5629                .collect::<Vec<_>>()
5630                .join(", ")
5631        );
5632        let repo = self.state.repo.clone();
5633        let coordination = self.coordinate(&repo).await?;
5634        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5635            self.state.note_withheld("fix", &r.withheld);
5636        }
5637        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5638        fix.committed = after != head_at_request;
5639        git::worktree_remove(&self.state.repo, &fix_worktree)
5640            .await
5641            .ok();
5642        drop(coordination);
5643
5644        self.state.event(
5645            "fix",
5646            match &fix.failed {
5647                Some(reason) => format!(
5648                    "operator fix: adoption report was lost ({reason}); {}",
5649                    if fix.committed {
5650                        "committed"
5651                    } else {
5652                        "NO new commit"
5653                    }
5654                ),
5655                None => format!(
5656                    "operator fix: {} addressed, {} rejected, {}",
5657                    fix.addressed.len(),
5658                    fix.rejected.len(),
5659                    if fix.committed {
5660                        "committed"
5661                    } else {
5662                        "NO new commit"
5663                    }
5664                ),
5665            },
5666        );
5667
5668        // Every selected finding gets an outcome — never left `Pending` once
5669        // the fixer's own turn is over. A report that never came back at all
5670        // marks every one of them `Unreported`, not silently "not addressed":
5671        // quota, a dropped stream, or an exhausted continuation are gaps in
5672        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5673        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5674        for f in &mut self.state.operator_fixes[request_index].findings {
5675            f.outcome = if fix.failed.is_some() {
5676                OperatorFixOutcome::Unreported
5677            } else if fix.addressed.contains(&f.id) {
5678                OperatorFixOutcome::Addressed
5679            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5680                OperatorFixOutcome::Rejected { why: r.why.clone() }
5681            } else {
5682                OperatorFixOutcome::Unreported
5683            };
5684        }
5685
5686        let committed = fix.committed;
5687        if committed {
5688            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5689        }
5690        self.state.operator_fixes[request_index].fix = Some(fix);
5691        // Saved again now that the fixer's own outcome is final, on top of
5692        // the save right after the request was first pushed above.
5693        self.state.save()?;
5694
5695        if committed {
5696            self.state.event(
5697                "fix",
5698                format!(
5699                    "operator fix committed {}; opening a follow-up review-only run",
5700                    short(&after)
5701                ),
5702            );
5703            // The operator asked for the fix, and the follow-up serves whatever
5704            // task the run it follows served.
5705            let origin =
5706                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5707            match Self::review(
5708                &self.state.repo,
5709                &winner.branch,
5710                self.state.config.clone(),
5711                origin,
5712            )
5713            .await
5714            {
5715                Ok(mut follow_up) => {
5716                    follow_up.state.event(
5717                        "start",
5718                        format!(
5719                            "requested by an operator fix on run {} for finding(s) {}",
5720                            self.state.id,
5721                            self.state.operator_fixes[request_index]
5722                                .findings
5723                                .iter()
5724                                .map(|f| f.id.as_str())
5725                                .collect::<Vec<_>>()
5726                                .join(", "),
5727                        ),
5728                    );
5729                    follow_up.state.save()?;
5730                    let follow_up_id = follow_up.state.id.clone();
5731                    // The follow-up is a run like any other: it belongs to the
5732                    // task of the run it follows, or to one filed for it.
5733                    let adopted = match crate::direct::adopt(
5734                        &follow_up.state,
5735                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5736                    ) {
5737                        Ok(a) => a,
5738                        Err(e) => {
5739                            // An ownerless run must not spend agent calls; it
5740                            // stays saved, and `magi run --resume` adopts it.
5741                            self.state.event(
5742                                "fix",
5743                                format!(
5744                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5745                                ),
5746                            );
5747                            self.state.save()?;
5748                            return Ok(());
5749                        }
5750                    };
5751                    let executed = follow_up.execute().await;
5752                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5753                    if let Some(a) = adopted {
5754                        a.finish(&follow_up.state, executed);
5755                    }
5756                    if let Some(e) = failure {
5757                        self.state.event(
5758                            "fix",
5759                            format!(
5760                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5761                            ),
5762                        );
5763                    }
5764                    self.state.operator_fixes[request_index].follow_up_review_run =
5765                        Some(follow_up_id);
5766                }
5767                Err(e) => {
5768                    self.state.event(
5769                        "fix",
5770                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5771                    );
5772                }
5773            }
5774            self.state.save()?;
5775        }
5776
5777        Ok(())
5778    }
5779
5780    // --------------------------------------------------------------- review
5781
5782    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5783    /// names, in order, each at most once, moving on only when the call
5784    /// advances (an error, a quota hit or nothing usable - the same decision
5785    /// point `agent::chain_advances` is for the other chained roles).
5786    ///
5787    /// Each agent gets its own seat from `Runner::seat`: the same agent
5788    /// continues its conversation (the winner's own implementer seat when it
5789    /// is the winner's author, now that the competition is over), another
5790    /// takes a fresh one so the full prompt is sent again. A handover is
5791    /// recorded under `node`, which is what makes the fallback stick for the
5792    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5793    /// is returned, so an exhausted chain reads exactly like a single failed
5794    /// fixer: one quota loss, the same wording, the same refund. An earlier
5795    /// attempt's edits are left in the tree and judged, with the final
5796    /// attempt's, by what git says afterwards.
5797    ///
5798    /// Returns the job that produced the outcome, for `continue_fix_report`.
5799    async fn ask_fixer(
5800        &mut self,
5801        winner: &Candidate,
5802        node: &'static str,
5803        round: Option<usize>,
5804        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5805    ) -> (SeatJob, SeatState, AgentOutcome) {
5806        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5807        self.ask_fixer_among(attempts, node, round, None, build)
5808            .await
5809    }
5810
5811    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5812    /// worktree and a commit: before the next agent is asked after one that
5813    /// did not satisfy the call, the worktree is put back on that commit, so
5814    /// a failed agent's partial edits are not carried into the next one.
5815    async fn ask_fixer_among(
5816        &mut self,
5817        attempts: Vec<(AgentSpec, String)>,
5818        node: &'static str,
5819        round: Option<usize>,
5820        restore: Option<(&Path, &str)>,
5821        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5822    ) -> (SeatJob, SeatState, AgentOutcome) {
5823        let run_id = self.state.id.clone();
5824        let prompts = self.state.config.prompts.clone();
5825        let cache = self.state.config.cache_dir();
5826        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5827        let mut last = None;
5828        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5829            let seat = self.seat(&key, &spec.id);
5830            let mut job = build(spec.clone(), seat);
5831            if i > 0 {
5832                job.stem = format!("{}-{}", job.stem, spec.id);
5833            }
5834            let ctx = WaveCtx {
5835                carry_seats: false,
5836                pauses: Some((&self.pause, &self.interrupt)),
5837                run: &run_id,
5838                node,
5839                prompts: &prompts,
5840                cache: cache.as_deref(),
5841                round,
5842            };
5843            let (seat, out) =
5844                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5845            // A resource block never advances the chain: the chosen agent
5846            // was not asked, so it must be the one asked again on resume.
5847            let advances = match &out {
5848                AgentOutcome::Ok(o) => agent::output_advances(o),
5849                AgentOutcome::ResourceBlocked(_) => false,
5850                _ => true,
5851            };
5852            if let Some(next) = ids.get(i + 1)
5853                && advances
5854            {
5855                self.state.seats.insert(seat.key.clone(), seat.clone());
5856                let class =
5857                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5858                let reason = match &out {
5859                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5860                    other => fail_reason(other),
5861                };
5862                // A failed agent's edits must not reach the next one, and a
5863                // restore that did not work ends the chain: the call fails
5864                // and the caller's own (confirmed) restore decides.
5865                if let Some((wt, rev)) = restore
5866                    && let Err(e) = self.graft_restore(wt, rev).await
5867                {
5868                    let why =
5869                        format!("could not restore the worktree before the next fixer: {e:#}");
5870                    tracing::warn!("{why}");
5871                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5872                        AgentOutcome::ResourceBlocked(why)
5873                    } else {
5874                        AgentOutcome::Failed(why)
5875                    };
5876                    last = Some((job, seat, out));
5877                    break;
5878                }
5879                record_handover(
5880                    &mut self.state,
5881                    node,
5882                    &seat.key,
5883                    &spec.id,
5884                    next,
5885                    &class,
5886                    &reason,
5887                );
5888                continue;
5889            }
5890            last = Some((job, seat, out));
5891            break;
5892        }
5893        last.expect("the fixer chain always has an entry")
5894    }
5895
5896    /// The one repair a verification-only failure earns (see
5897    /// [`crate::recovery`]), before the first review round of a run that took
5898    /// a branch over.
5899    ///
5900    /// Saved as started *before* the fixer is called, so a resume never
5901    /// repairs twice; a start found on re-entry is rolled back to the commit
5902    /// it began from. A failed or unusable fixer leaves the tree as it was and
5903    /// the review goes on over it, so the first round's own verification
5904    /// reports the failure again. The failure is never carried into a round as
5905    /// a result: verification is re-run on whatever commit this leaves.
5906    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5907        use crate::recovery::{Mode, Repair};
5908        if !self.state.reviews.is_empty() {
5909            return Ok(());
5910        }
5911        let Some(mut rec) = self.state.recovery.clone() else {
5912            return Ok(());
5913        };
5914        if rec.mode != Mode::VerifyRepair {
5915            return Ok(());
5916        }
5917        let wt = winner.worktree.clone();
5918        let mut head = git::rev_parse(&wt, "HEAD").await?;
5919        match rec.repair.clone() {
5920            Repair::Done { .. } => return Ok(()),
5921            Repair::WaitingForRestore { before } => {
5922                self.graft_restore(&wt, &before).await?;
5923                head = before;
5924                rec.repair = Repair::NotStarted;
5925                self.state.recovery = Some(rec.clone());
5926                self.state.save()?;
5927            }
5928            Repair::Started { before } => {
5929                // Interrupted: the unfinished edits are not trusted.
5930                // An unconfirmed rollback blocks the run with `Started` kept,
5931                // so a resume retries it; the tree is never reviewed as is.
5932                if let Err(e) = self.graft_restore(&wt, &before).await {
5933                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5934                        return Err(e);
5935                    }
5936                    return self
5937                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5938                        .await;
5939                }
5940                rec.repair = Repair::Done {
5941                    before: before.clone(),
5942                    after: before,
5943                    committed: false,
5944                    failed: Some("interrupted; rolled back".to_owned()),
5945                };
5946                self.state
5947                    .event("recovery", "the repair was interrupted and rolled back");
5948                self.state.recovery = Some(rec);
5949                self.state.save()?;
5950                return Ok(());
5951            }
5952            Repair::NotStarted => {}
5953        }
5954        if crate::recovery::recheck(&mut rec, &head) {
5955            self.state.event("recovery", rec.summary());
5956            self.state.recovery = Some(rec);
5957            self.state.save()?;
5958            return Ok(());
5959        }
5960        rec.repair = Repair::Started {
5961            before: head.clone(),
5962        };
5963        self.state.recovery = Some(rec.clone());
5964        self.state.save()?;
5965
5966        let language = self.state.config.graph.language.clone();
5967        let sessions = self.state.config.graph.sessions;
5968        let artifacts = agent::artifacts_dir(&self.state.dir());
5969        let fix_prompt = prompt::recovery_fix(
5970            &self.state.instruction,
5971            &rec,
5972            self.state.config.graph.review_rounds,
5973            &language,
5974        );
5975        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5976        self.state.event(
5977            "recovery",
5978            format!(
5979                "repair: the fixer starts from the failed verification on {}",
5980                short(&head)
5981            ),
5982        );
5983        let (_job, seat, out) = self
5984            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5985                prompt: fix_prompt.clone(),
5986                spec,
5987                seat,
5988                cwd: wt.clone(),
5989                timeout,
5990                allow_write: true,
5991                sessions,
5992                artifacts: artifacts.clone(),
5993                stem: "fix-recovery".to_owned(),
5994                handover: None,
5995            })
5996            .await;
5997        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5998            rec.repair = Repair::WaitingForRestore { before: head };
5999            self.state.recovery = Some(rec);
6000            return self.stop_for_resource();
6001        }
6002        let failed = match &out {
6003            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6004            AgentOutcome::Quota(o) => {
6005                self.state.quota.push(QuotaLoss {
6006                    seat: seat.key.clone(),
6007                    node: "fix".to_owned(),
6008                    at: Timestamp::now(),
6009                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6010                });
6011                Some("rate limited (quota); the fixer could not run".to_owned())
6012            }
6013            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6014            AgentOutcome::Failed(e) => Some(e.clone()),
6015        };
6016        self.state.seats.insert(seat.key.clone(), seat);
6017        let coordination = self.coordinate(&wt).await?;
6018        let failed = match failed {
6019            // The agent's partial edits are not kept.
6020            Some(why) => {
6021                if let Err(e) = git::restore_to(&wt, &head).await {
6022                    return self
6023                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6024                        .await;
6025                }
6026                Some(why)
6027            }
6028            None => {
6029                if let Ok(r) = git::rescue_commit(
6030                    &wt,
6031                    "magi: recovery repair of the failed verification (uncommitted work)",
6032                )
6033                .await
6034                {
6035                    self.state.note_withheld("fix", &r.withheld);
6036                }
6037                None
6038            }
6039        };
6040        let after = git::rev_parse(&wt, "HEAD").await?;
6041        drop(coordination);
6042        self.state.event(
6043            "recovery",
6044            match &failed {
6045                Some(why) => format!("repair: nothing kept ({why})"),
6046                None if after != head => {
6047                    format!("repair: committed {}", short(&after))
6048                }
6049                None => "repair: the fixer made no commit".to_owned(),
6050            },
6051        );
6052        rec.repair = Repair::Done {
6053            before: head.clone(),
6054            committed: after != head,
6055            after,
6056            failed,
6057        };
6058        self.state.recovery = Some(rec);
6059        self.state.save()?;
6060        Ok(())
6061    }
6062
6063    /// A repair whose rollback could not be confirmed: the run is blocked and
6064    /// resumable, and the unconfirmed state is never reviewed.
6065    async fn stop_recovery(
6066        &mut self,
6067        mut rec: crate::recovery::RecoveryRecord,
6068        head: &str,
6069        why: String,
6070    ) -> Result<()> {
6071        // `Started` stays recorded: a resume retries the restore.
6072        rec.repair = crate::recovery::Repair::Started {
6073            before: head.to_owned(),
6074        };
6075        self.state.recovery = Some(rec);
6076        self.state
6077            .event("recovery", format!("repair could not be undone: {why}"));
6078        self.state.status = RunStatus::Blocked;
6079        self.state.save()?;
6080        bail!("recovery repair could not be rolled back: {why}")
6081    }
6082
6083    async fn review_loop(&mut self) -> Result<()> {
6084        if self.hold_base_sync().await? {
6085            return Ok(());
6086        }
6087        loop {
6088            if self
6089                .state
6090                .base_sync
6091                .as_ref()
6092                .is_some_and(|s| s.conflict.is_some())
6093            {
6094                return Ok(());
6095            }
6096            if self.catchup_repairable() {
6097                let record = self.state.reviews.last().expect("repairable round").clone();
6098                let winner = self
6099                    .state
6100                    .winner()
6101                    .cloned()
6102                    .context("catch-up repair has no winner")?;
6103                if !self.fix_review_round(&winner, record).await? {
6104                    return Ok(());
6105                }
6106            }
6107            self.review_loop_pass().await?;
6108            if !self.catchup_repairable() {
6109                return Ok(());
6110            }
6111        }
6112    }
6113
6114    /// Derive an owed catch-up repair from durable round evidence, including
6115    /// reentry after a crash before the fixer was dispatched.
6116    fn catchup_repairable(&self) -> bool {
6117        self.state.reviews.len() < self.state.config.graph.review_rounds
6118            && self.state.reviews.last().is_some_and(|r| {
6119                !r.clean
6120                    && r.blocking == 0
6121                    && r.fix.is_none()
6122                    && r.e2e_status() == E2eStatus::Failed
6123            })
6124    }
6125
6126    async fn review_loop_pass(&mut self) -> Result<()> {
6127        // A base that would not rebase is a person's decision, not a review
6128        // round: nothing here would change the answer, and reviewers and a
6129        // fixer would be spending real budget on a tree that cannot land
6130        // regardless of what they find.
6131        if self
6132            .state
6133            .base_sync
6134            .as_ref()
6135            .is_some_and(|s| s.conflict.is_some())
6136        {
6137            return Ok(());
6138        }
6139        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6140        // agent files with `magi task add` name the run that paid for it. The
6141        // prompt overlay is cloned alongside it because the waves borrow it
6142        // while `self` is mutably borrowed by the node's own bookkeeping.
6143        let run_id = self.state.id.clone();
6144        let prompts = self.state.config.prompts.clone();
6145        let Some(winner) = self.state.winner().cloned() else {
6146            return Ok(());
6147        };
6148        let max_rounds = self.state.config.graph.review_rounds;
6149        // A clean round, an exhausted round budget, or a stalled tree (see
6150        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6151        // they are recorded — recomputed here, not read off `status`, so a
6152        // reentry into a run that already stopped restates the identical
6153        // verdict instead of silently handing back whatever an earlier node
6154        // in this same walk clobbered `status` to (a solo-candidate
6155        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6156        // below runs an empty range once the budget is spent, and would
6157        // otherwise fall through without touching `status` at all.
6158        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6159            // A reentry after a crash between the last round's save and
6160            // `stop_reviewing` reaches the hand-off here, not there.
6161            if status == RunStatus::Gating {
6162                self.record_contested_handoff();
6163            }
6164            self.state.status = status;
6165            self.state.save()?;
6166            return Ok(());
6167        }
6168        self.state.status = RunStatus::Reviewing;
6169        // A last recorded round whose own verification never resolved
6170        // (`ResourceBlocked` — the shared build cache, not the patch) is
6171        // never a concluded round, whatever the round budget says: starting
6172        // a fresh round on top of it would spend a whole new reviewer wave
6173        // re-reading an unchanged patch instead of just retrying the one
6174        // check that actually needs it, and once the budget is spent the
6175        // loop below has nothing left to do at all (its range is empty).
6176        // Retry that check directly instead, exactly the same retry
6177        // `stop_reviewing` already does for its own catch-up case.
6178        if self.state.reviews.last().is_some_and(|r| {
6179            r.e2e_status() == E2eStatus::ResourceBlocked
6180                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6181        }) {
6182            let shell = self.state.config.shell();
6183            return self
6184                .stop_unresolved_review(
6185                    "the last round's own verification never resolved",
6186                    &shell,
6187                    &winner.worktree,
6188                )
6189                .await;
6190        }
6191
6192        self.recovery_repair(&winner).await?;
6193        if self.resource_halt()? {
6194            return Ok(());
6195        }
6196
6197        let repo = self.state.repo.clone();
6198        let root = self.state.worktree_root();
6199        let language = self.state.config.graph.language.clone();
6200        let sessions = self.state.config.graph.sessions;
6201        let artifacts = self.merge_repair_artifacts();
6202        let base = self.landing_base();
6203        let base_short = short(&base);
6204        let reviewers = self.roles.reviewers.clone();
6205        let shell = self.state.config.shell();
6206
6207        for round in (self.state.reviews.len() + 1)..=max_rounds {
6208            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6209            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6210            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6211            // The prior round's own record, already persisted — never a
6212            // hand-carried variable of just its failing output: that is
6213            // exactly what let a round's e2e result drift out of sync with
6214            // which commit it was actually about (see `SCHEMA`'s doc for
6215            // schema 8). Judged against `head`, the commit reviewers are
6216            // about to look at now, so the summary always reads as "an
6217            // earlier head" here — this round's own patch has not been
6218            // checked yet.
6219            let prev_verification = self
6220                .state
6221                .reviews
6222                .last()
6223                .and_then(|r| r.verification_summary(&head));
6224            // Only the first round of a run that took a branch over: later
6225            // rounds have this run's own history.
6226            let prior_section = match (&self.state.recovery, round) {
6227                (Some(rec), 1) if self.state.reviews.is_empty() => {
6228                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6229                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6230                            Some(crate::recovery::head_of(
6231                                &git::diff(&winner.worktree, reviewed, "HEAD")
6232                                    .await
6233                                    .unwrap_or_default(),
6234                                crate::recovery::SINCE_DIFF_MAX,
6235                            ))
6236                        }
6237                        _ => None,
6238                    };
6239                    Some(prompt::recovery_review(rec, since.as_deref()))
6240                }
6241                _ => None,
6242            };
6243
6244            // Each reviewer gets its own detached checkout of exactly this
6245            // commit: nobody can perturb the winner's tree, and the fixer can
6246            // keep working without racing a reviewer.
6247            let mut jobs = Vec::new();
6248            let coordination = self.coordinate(&repo).await?;
6249            for (r, spec) in reviewers.iter().cloned().enumerate() {
6250                let wt = root.join(format!("review-{}", r + 1));
6251                // `clean -fdx` in a reset wipes setup's products, so every
6252                // round sets the seat up again.
6253                if wt.exists() {
6254                    git::reset_detached(&wt, &head).await?;
6255                } else {
6256                    git::worktree_add_detached(&repo, &wt, &head).await?;
6257                }
6258                self.setup_seat_worktree(&repo, &wt, false).await?;
6259                let seat_key = format!("review-{}", r + 1);
6260                // The seat starts the round on whoever answered it last, not
6261                // on the agent the spec names, so a failure is not re-paid.
6262                let spec = pick_start_spec(
6263                    &self.roles.reviewer_roster,
6264                    spec,
6265                    self.state.seat_history.get(&seat_key),
6266                );
6267                let seat = self.seat(&seat_key, &spec.id);
6268                jobs.push(SeatJob {
6269                    prompt: prompt::review(&prompt::ReviewCtx {
6270                        instruction: &self.state.instruction,
6271                        branch: &winner.branch,
6272                        base_short: &base_short,
6273                        stat: &stat,
6274                        patch: &patch,
6275                        verification: prev_verification.as_ref(),
6276                        prior: prior_section.as_deref(),
6277                        reviewers: reviewers.len(),
6278                        round,
6279                        rounds: max_rounds,
6280                        // A review-only run has no rankings, so nothing
6281                        // competed for this patch and the reviewer is told so.
6282                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6283                        lens: Lens::for_seat(r),
6284                        language: &language,
6285                        github_language: &self.state.config.graph.github_language,
6286                    }),
6287                    spec,
6288                    seat,
6289                    cwd: wt,
6290                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6291                    allow_write: false,
6292                    sessions,
6293                    artifacts: artifacts.clone(),
6294                    stem: format!("review-{round}-{}", r + 1),
6295                    handover: None,
6296                });
6297            }
6298
6299            drop(coordination);
6300            self.state.event(
6301                "review",
6302                format!(
6303                    "round {round}: {} reviewers on {}",
6304                    jobs.len(),
6305                    short(&head)
6306                ),
6307            );
6308            let mut quota_losses = Vec::new();
6309            let review_retries = self.state.config.graph.retries;
6310            let review_cache = self.state.config.cache_dir();
6311            let ctx = WaveCtx {
6312                carry_seats: true,
6313                pauses: Some((&self.pause, &self.interrupt)),
6314                run: &run_id,
6315                node: "review",
6316                prompts: &prompts,
6317                cache: review_cache.as_deref(),
6318                round: Some(round),
6319            };
6320            let results = ask_json_wave::<Review>(
6321                jobs,
6322                Arc::clone(&self.sem),
6323                review_retries,
6324                &self.roles.reviewer_roster,
6325                &ctx,
6326                &mut quota_losses,
6327                &mut self.state,
6328                &|_: &Review| Ok(()),
6329            )
6330            .await;
6331            if self.state.resource_block.is_some() {
6332                return self.stop_for_resource();
6333            }
6334            // Counted before the move below: how many of *this* round's
6335            // reviewer seats were lost to their own rate limit, as opposed to
6336            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6337            let round_quota_missing = quota_losses.len();
6338            self.state.quota.extend(quota_losses);
6339
6340            let mut records = Vec::new();
6341            let mut all_findings = Vec::new();
6342            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6343                let agent_id = seat.agent.clone();
6344                self.state.seats.insert(seat.key.clone(), seat);
6345                let mut record = ReviewRecord {
6346                    reviewer: r + 1,
6347                    agent: agent_id,
6348                    summary: String::new(),
6349                    findings: Vec::new(),
6350                    vote: None,
6351                    failed: None,
6352                    duration_ms: 0,
6353                    // Set for both outcomes: `failed: Some(_)` with
6354                    // `attempts > 0` is a seat every retry still lost, not a
6355                    // recovered one — only `failed: None` with `attempts > 0`
6356                    // reads as "answered after a nudge" (see this field's own
6357                    // doc).
6358                    attempts,
6359                };
6360                match res {
6361                    Ok((review, out)) => {
6362                        // Sanitized here, at the point every other piece of
6363                        // agent prose in this file is (candidate summaries,
6364                        // deliberation turns, vote reasons): a reviewer's own
6365                        // words are the one thing about it that could name
6366                        // it, and reconsideration below broadcasts this same
6367                        // summary and these same findings to every other
6368                        // seat on the panel.
6369                        record.summary =
6370                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6371                        record.vote = Some(review.vote);
6372                        record.duration_ms = out.duration_ms;
6373                        for (n, mut f) in review.findings.into_iter().enumerate() {
6374                            // ids are magi's, never the agent's: the fixer's
6375                            // adoption report is keyed by them.
6376                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6377                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6378                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6379                            // `file` is agent-supplied prose too, never
6380                            // checked against the real tree — the same
6381                            // exposure `title`/`detail` above have, just in
6382                            // a field easy to forget because it looks like a
6383                            // path rather than free text.
6384                            f.file = f
6385                                .file
6386                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6387                            all_findings.push(f.clone());
6388                            record.findings.push(f);
6389                        }
6390                        self.state.event(
6391                            "review",
6392                            format!(
6393                                "round {round}: reviewer {} voted {} with {} finding(s)",
6394                                r + 1,
6395                                review.vote.label(),
6396                                record.findings.len()
6397                            ),
6398                        );
6399                    }
6400                    Err(e) => {
6401                        record.failed = Some(e.to_string());
6402                        self.state.event(
6403                            "review",
6404                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6405                        );
6406                    }
6407                }
6408                records.push(record);
6409            }
6410
6411            // Tally the round's votes and, if they split, spend the one
6412            // round of reconsideration the split -> deliberate -> revote
6413            // shape `judge`/`vote` use for the panel, sized down to what a
6414            // read-only review round can afford: one round, and a revote
6415            // rather than an argument, because the panel already wrote its
6416            // reasoning down as findings the first time around.
6417            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6418            let vote_split =
6419                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6420            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6421            if vote_split {
6422                self.state.event(
6423                    "review",
6424                    format!(
6425                        "round {round}: votes split ({}) — one round of reconsideration",
6426                        initial_votes
6427                            .iter()
6428                            .map(|v| v.label())
6429                            .collect::<Vec<_>>()
6430                            .join(", ")
6431                    ),
6432                );
6433                // Seats read every seat's findings and votes, still numbered
6434                // and never named — the same anonymity `review` itself keeps.
6435                let panel: Vec<ReviewSeatReport<'_>> = records
6436                    .iter()
6437                    .filter_map(|r| {
6438                        r.vote.map(|vote| ReviewSeatReport {
6439                            reviewer: r.reviewer,
6440                            vote,
6441                            summary: &r.summary,
6442                            findings: &r.findings,
6443                        })
6444                    })
6445                    .collect();
6446
6447                let mut jobs = Vec::new();
6448                let mut seats_at = Vec::new();
6449                for (r, spec) in reviewers.iter().cloned().enumerate() {
6450                    // A seat with no initial vote has nothing to reconsider
6451                    // from and stays absent, the same as it stayed absent
6452                    // from `panel` above.
6453                    if records[r].vote.is_none() {
6454                        continue;
6455                    }
6456                    let wt = root.join(format!("review-{}", r + 1));
6457                    let seat_key = format!("review-{}", r + 1);
6458                    let spec = self.occupant(&seat_key, spec);
6459                    let seat = self.seat(&seat_key, &spec.id);
6460                    // A seat with no live session has already forgotten the
6461                    // initial review's prompt — restate the patch it is
6462                    // voting on, the same as `deliberate`/`vote` do for a
6463                    // judge in the same position.
6464                    // The panel already carries this seat's own review and
6465                    // vote, so restating the patch makes the prompt whole for
6466                    // a seat handed to another agent.
6467                    let build = |with_patch: bool| {
6468                        prompt::review_reconsider(&ReviewReconsiderCtx {
6469                            instruction: &self.state.instruction,
6470                            reviewer: r + 1,
6471                            lens: Lens::for_seat(r),
6472                            panel: &panel,
6473                            patch: with_patch.then_some(ReviewPatch {
6474                                branch: &winner.branch,
6475                                base_short: &base_short,
6476                                stat: &stat,
6477                                patch: &patch,
6478                            }),
6479                            round,
6480                            rounds: max_rounds,
6481                            language: &language,
6482                        })
6483                    };
6484                    let full = build(true);
6485                    let prompt = if has_context(&spec, &seat, sessions) {
6486                        build(false)
6487                    } else {
6488                        full.clone()
6489                    };
6490                    jobs.push(SeatJob {
6491                        prompt,
6492                        spec,
6493                        seat,
6494                        cwd: wt,
6495                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6496                        allow_write: false,
6497                        sessions,
6498                        artifacts: artifacts.clone(),
6499                        stem: format!("review-{round}-reconsider-{}", r + 1),
6500                        handover: Some(full),
6501                    });
6502                    seats_at.push(r);
6503                }
6504
6505                let mut recon_quota_losses = Vec::new();
6506                let recon_cache = self.state.config.cache_dir();
6507                let recon_ctx = WaveCtx {
6508                    carry_seats: true,
6509                    pauses: Some((&self.pause, &self.interrupt)),
6510                    run: &run_id,
6511                    node: "review",
6512                    prompts: &prompts,
6513                    cache: recon_cache.as_deref(),
6514                    round: Some(round),
6515                };
6516                let recon_results = ask_json_wave::<ReviewRevote>(
6517                    jobs,
6518                    Arc::clone(&self.sem),
6519                    review_retries,
6520                    &self.roles.reviewer_roster,
6521                    &recon_ctx,
6522                    &mut recon_quota_losses,
6523                    &mut self.state,
6524                    &|_: &ReviewRevote| Ok(()),
6525                )
6526                .await;
6527                if self.state.resource_block.is_some() {
6528                    return self.stop_for_resource();
6529                }
6530                self.state.quota.extend(recon_quota_losses);
6531
6532                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6533                    let agent_id = seat.agent.clone();
6534                    self.state.seats.insert(seat.key.clone(), seat);
6535                    let mut rec = ReviewRevoteRecord {
6536                        reviewer: r + 1,
6537                        agent: agent_id,
6538                        vote: None,
6539                        reason: String::new(),
6540                        failed: None,
6541                    };
6542                    match res {
6543                        Ok((rv, _)) => {
6544                            rec.vote = Some(rv.vote);
6545                            rec.reason =
6546                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6547                            self.state.event(
6548                                "review",
6549                                format!(
6550                                    "round {round}: reviewer {} revoted {}",
6551                                    r + 1,
6552                                    rv.vote.label()
6553                                ),
6554                            );
6555                        }
6556                        Err(e) => {
6557                            rec.failed = Some(e.to_string());
6558                            self.state.event(
6559                                "review",
6560                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6561                            );
6562                        }
6563                    }
6564                    reconsideration.push(rec);
6565                }
6566            } else if initial_votes.len() > 1 {
6567                self.state.event(
6568                    "review",
6569                    format!(
6570                        "round {round}: votes agreed ({}) — no reconsideration",
6571                        initial_votes[0].label()
6572                    ),
6573                );
6574            }
6575
6576            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6577            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6578            // A round that already has a blocking finding and a round left to
6579            // try is going back to the fixer no matter what `verify.e2e`
6580            // says, so running it first only spends the loop's slowest step
6581            // (minutes, for a Rust repo's full test suite) on a head about
6582            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6583            // runs once a round has no blocking findings left (see
6584            // `round_is_clean`, which a deferred — empty — `e2e` can never
6585            // satisfy since `blocking` is nonzero whenever this branch is
6586            // taken), and `stop_reviewing` forces a real run before it will
6587            // ever read a deferred round as green.
6588            let defer_e2e =
6589                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6590            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6591                let reason =
6592                    format!("{blocking} blocking finding(s) already required a fix this round");
6593                self.state.event(
6594                    "verify",
6595                    format!(
6596                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6597                         {}); it will run once a round has none left",
6598                        short(&head)
6599                    ),
6600                );
6601                (Vec::new(), false, true, Some(reason))
6602            } else {
6603                let e2e_commands = self.state.config.verify.e2e.clone();
6604                let cache_dir = self.state.config.cache_dir();
6605                let context = format!("round {round}");
6606                let wait = self.repository_wait();
6607                let (e2e, verify_retried) = with_cache_lease(
6608                    &mut self.state,
6609                    cache_dir.as_deref(),
6610                    "e2e",
6611                    "e2e",
6612                    &winner.worktree,
6613                    &head,
6614                    verify_timeout,
6615                    &context,
6616                    &wait,
6617                    |state, budget| {
6618                        let shell = shell.clone();
6619                        let e2e_commands = e2e_commands.clone();
6620                        let worktree = winner.worktree.clone();
6621                        let context = context.clone();
6622                        async move {
6623                            run_e2e_with_retry(
6624                                state,
6625                                &shell,
6626                                &e2e_commands,
6627                                &worktree,
6628                                budget,
6629                                &context,
6630                            )
6631                            .await
6632                        }
6633                    },
6634                )
6635                .await?;
6636                (e2e, verify_retried, false, None)
6637            };
6638
6639            let expected = records.len();
6640            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6641            let incomplete = answered < expected;
6642            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6643            let policy = self.state.config.graph.incomplete_review;
6644            let clean = round_is_clean(
6645                blocking,
6646                e2e_ok,
6647                answered,
6648                expected,
6649                round_quota_missing,
6650                policy,
6651            );
6652
6653            let mut round_record = ReviewRound {
6654                round,
6655                head: head.clone(),
6656                verified_head: None,
6657                verified_at: None,
6658                reviews: records,
6659                e2e,
6660                verify_retried,
6661                e2e_deferred,
6662                e2e_defer_reason,
6663                fix: None,
6664                blocking,
6665                answered,
6666                expected,
6667                clean,
6668                progressed: false,
6669                vote_split,
6670                reconsideration,
6671                verdict: None,
6672            };
6673            // The final vote per seat is its revote where reconsideration
6674            // ran and answered, its initial vote otherwise — the same
6675            // fallback `tally` uses for a judge whose private vote failed.
6676            round_record.verdict = ReviewVote::worst(
6677                round_record
6678                    .final_votes()
6679                    .into_iter()
6680                    .map(|(_, _, vote)| vote),
6681            );
6682            // Which commit and when magi actually attempted to check —
6683            // known the moment a command was dispatched against `head`,
6684            // whether or not it finished: a resource-blocked attempt still
6685            // targeted a specific commit at a specific time, and leaving
6686            // that unrecorded is exactly what made `verification_summary`
6687            // report a fresh attempt as "commit unknown ... recorded before
6688            // this was tracked", indistinguishable from a genuinely old,
6689            // untracked record. Only a deferred or unconfigured round never
6690            // ran at all and has nothing to record — see
6691            // `ReviewRound::verified_head`'s own doc.
6692            if !matches!(
6693                round_record.e2e_status(),
6694                E2eStatus::Deferred | E2eStatus::NotConfigured
6695            ) {
6696                round_record.verified_head = Some(head.clone());
6697                round_record.verified_at = Some(Timestamp::now());
6698            }
6699
6700            if incomplete {
6701                let missing: Vec<String> = round_record
6702                    .reviews
6703                    .iter()
6704                    .filter(|r| r.failed.is_some())
6705                    .map(|r| format!("review-{}", r.reviewer))
6706                    .collect();
6707                self.state.event(
6708                    "review",
6709                    format!(
6710                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6711                        missing.join(", ")
6712                    ),
6713                );
6714            }
6715
6716            if clean {
6717                self.state.event(
6718                    "review",
6719                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6720                        format!(
6721                            "round {round}: clean (warn policy, incomplete panel) — no \
6722                             blocking findings from the seats that answered, verification green"
6723                        )
6724                    } else if incomplete {
6725                        format!(
6726                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6727                             quorum) — no blocking findings from the seats that answered, \
6728                             verification green",
6729                            expected - answered
6730                        )
6731                    } else {
6732                        format!("round {round}: clean — no blocking findings, verification green")
6733                    },
6734                );
6735                self.state.reviews.push(round_record);
6736                self.state.status = RunStatus::Gating;
6737                self.state.save()?;
6738                return Ok(());
6739            }
6740
6741            // Nothing was raised and verification passed, but not every seat
6742            // answered and `round_is_clean` still refused to call it clean —
6743            // either a seat is missing for a reason other than its own quota
6744            // (a crash, a timeout, unparsable output — worth another try), or
6745            // every seat that could have answered lost its quota and nobody
6746            // is left to decide on: re-review rather than send the fixer
6747            // after a round with nothing to fix.
6748            if incomplete && blocking == 0 && e2e_ok {
6749                self.state.reviews.push(round_record);
6750                self.state.save()?;
6751                if round == max_rounds {
6752                    self.state.status = RunStatus::Blocked;
6753                    self.state.event(
6754                        "review",
6755                        format!(
6756                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6757                             refusing to call it clean",
6758                            expected - answered
6759                        ),
6760                    );
6761                    return Ok(());
6762                }
6763                continue;
6764            }
6765
6766            // Nothing for the fixer to act on (`blocking == 0`) and the only
6767            // reason this round is not clean is that magi itself never got
6768            // a command to run — the shared build cache, not the patch (see
6769            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6770            // the fixer would invite a change to appease contention that has
6771            // nothing to do with the diff, and would leave this attempt
6772            // sitting in the next round's prompt as if it were about an
6773            // earlier, superseded commit rather than what it actually is:
6774            // the same head, still waiting to be checked. Wait for it the
6775            // same way the final round's own contention is already handled,
6776            // whatever round this happens to be.
6777            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6778                self.state.reviews.push(round_record);
6779                return self
6780                    .stop_unresolved_review(
6781                        "the round's own verification could not run",
6782                        &shell,
6783                        &winner.worktree,
6784                    )
6785                    .await;
6786            }
6787
6788            if round == max_rounds {
6789                self.state.reviews.push(round_record);
6790                return self
6791                    .stop_reviewing(
6792                        &format!(
6793                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6794                        ),
6795                        &shell,
6796                        &winner.worktree,
6797                    )
6798                    .await;
6799            }
6800
6801            if !self.fix_review_round(&winner, round_record).await? {
6802                return Ok(());
6803            }
6804        }
6805        Ok(())
6806    }
6807
6808    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6809    /// False means admission or continuation policy stopped the review loop.
6810    async fn fix_review_round(
6811        &mut self,
6812        winner: &Candidate,
6813        mut round_record: ReviewRound,
6814    ) -> Result<bool> {
6815        let round = round_record.round;
6816        let shell = self.state.config.shell();
6817        let step = self.run_fix_round(winner, &mut round_record).await?;
6818        if matches!(step, FixStep::Resource) {
6819            self.stop_for_resource()?;
6820            return Ok(false);
6821        }
6822        if self.state.reviews.len() < round {
6823            self.state.reviews.push(round_record);
6824        } else {
6825            self.state.reviews[round - 1] = round_record;
6826        }
6827        self.state.save()?;
6828
6829        // The fixer's own report never came back this round, even after
6830        // `continue_fix_report`'s own budget was spent on it — not an
6831        // ordinary "no report" (dropped stream, quota, plain failure),
6832        // which already reads that way and is left to the existing round
6833        // budget. Stopping here, rather than opening another round, is
6834        // what keeps a next reviewer/fixer wave from ever being
6835        // dispatched onto `winner.worktree` while whatever the seat's
6836        // last call may still have running there is unaccounted for: no
6837        // process liveness check exists (and none is being added — see
6838        // AGENTS.md/this task's own scope), so the only way to honour
6839        // "nothing starts before a valid report returns" is to not start
6840        // anything further on this worktree from this run at all.
6841        if matches!(step, FixStep::Recorded { report_lost: true }) {
6842            self.stop_reviewing(
6843                "the fixer's adoption report never came back, even after resuming its \
6844                         own seat; refusing to start another round against the same worktree \
6845                         while that is unresolved",
6846                &shell,
6847                &winner.worktree,
6848            )
6849            .await?;
6850            return Ok(false);
6851        }
6852
6853        let streak = self
6854            .state
6855            .reviews
6856            .iter()
6857            .rev()
6858            .take_while(|r| !r.progressed)
6859            .count();
6860        if streak >= STAGNANT_LIMIT {
6861            self.stop_reviewing(
6862                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6863                &shell,
6864                &winner.worktree,
6865            )
6866            .await?;
6867            return Ok(false);
6868        }
6869        Ok(true)
6870    }
6871
6872    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6873    async fn run_fix_round(
6874        &mut self,
6875        winner: &Candidate,
6876        round_record: &mut ReviewRound,
6877    ) -> Result<FixStep> {
6878        let round = round_record.round;
6879        let max_rounds = self.state.config.graph.review_rounds;
6880        let run_id = self.state.id.clone();
6881        let prompts = self.state.config.prompts.clone();
6882        let language = self.state.config.graph.language.clone();
6883        let sessions = self.state.config.graph.sessions;
6884        let artifacts = self.merge_repair_artifacts();
6885        let base = self.landing_base();
6886        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6887        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6888        let this_round_verification = round_record.verification_summary(&head);
6889        let all_findings: Vec<_> = round_record
6890            .reviews
6891            .iter()
6892            .flat_map(|r| r.findings.iter().cloned())
6893            .collect();
6894        // Fix. The winner's own implementer seat continues its conversation:
6895        // the competition is over, so context is pure benefit now.
6896        let blocking_findings: Vec<_> = all_findings
6897            .iter()
6898            .filter(|f| f.severity.blocks())
6899            .cloned()
6900            .collect();
6901        let fix_prompt = prompt::fix_in(
6902            &self.state.instruction,
6903            &blocking_findings,
6904            this_round_verification.as_ref(),
6905            round,
6906            max_rounds,
6907            &language,
6908            &self.state.config.graph.github_language,
6909        );
6910        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6911        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6912        let (job, seat, out) = self
6913            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6914                prompt: fix_prompt.clone(),
6915                spec,
6916                seat,
6917                cwd: winner.worktree.clone(),
6918                timeout,
6919                allow_write: true,
6920                sessions,
6921                artifacts: artifacts.clone(),
6922                stem: format!("fix-{round}"),
6923                handover: None,
6924            })
6925            .await;
6926        // Not asked: the round is not recorded (so a resume repeats it),
6927        // no fixer was handed over to, and the seat is left as it was.
6928        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6929            return Ok(FixStep::Resource);
6930        }
6931        let agent_id = seat.agent.clone();
6932
6933        let mut fix = FixRecord {
6934            agent: agent_id,
6935            addressed: Vec::new(),
6936            rejected: Vec::new(),
6937            notes: String::new(),
6938            committed: false,
6939            failed: None,
6940            duration_ms: 0,
6941            continuation: None,
6942        };
6943        let mut continuation = ContinuationRecord::not_needed();
6944        let mut final_seat = seat.clone();
6945        match out {
6946            AgentOutcome::Ok(o) => {
6947                fix.duration_ms = o.duration_ms;
6948                let parsed = verdict::extract_json::<FixReport>(&o.text);
6949                // A parsed report standing next to a command this same
6950                // reply's own CLI never confirmed the exit status of is
6951                // not a resolved answer — the identical `CommandEvidence`
6952                // `state.jobs` renders, read here instead of only on
6953                // display, per the completion judgment and the shown
6954                // record needing to agree.
6955                let incomplete_reason = match &parsed {
6956                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6957                        "the reply parsed, but it reported a command whose own CLI never \
6958                             confirmed an exit status"
6959                            .to_owned(),
6960                    ),
6961                    Ok(_) => None,
6962                    Err(e) => Some(e.to_string()),
6963                };
6964                match incomplete_reason {
6965                    None => {
6966                        let report = parsed.expect("checked Ok above");
6967                        fix.addressed = report.addressed;
6968                        fix.rejected = report.rejected;
6969                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6970                    }
6971                    Some(reason) => {
6972                        let (resumed_seat, resolved, failure, cont) = self
6973                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6974                            .await;
6975                        fix.duration_ms += cont.cumulative_wait_ms;
6976                        continuation = cont;
6977                        final_seat = resumed_seat;
6978                        match resolved {
6979                            Some(report) => {
6980                                fix.addressed = report.addressed;
6981                                fix.rejected = report.rejected;
6982                                fix.notes =
6983                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6984                            }
6985                            None => fix.failed = failure,
6986                        }
6987                    }
6988                }
6989            }
6990            // The CLI's raw error JSON is not a fix report to parse.
6991            AgentOutcome::Dropped(o) => {
6992                fix.duration_ms = o.duration_ms;
6993                let why = o
6994                    .dropped
6995                    .as_ref()
6996                    .map(|d| d.why.as_str())
6997                    .unwrap_or("the CLI ended the stream without delivering its answer");
6998                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6999            }
7000            AgentOutcome::Quota(o) => {
7001                self.state.quota.push(QuotaLoss {
7002                    seat: final_seat.key.clone(),
7003                    node: "fix".to_owned(),
7004                    at: Timestamp::now(),
7005                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7006                });
7007                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7008            }
7009            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7010        }
7011        fix.continuation = Some(continuation);
7012        self.state.seats.insert(final_seat.key.clone(), final_seat);
7013        let coordination = self.coordinate(&winner.worktree).await?;
7014        if let Ok(r) = git::rescue_commit(
7015            &winner.worktree,
7016            &format!("magi: review round {round} fixes (uncommitted work)"),
7017        )
7018        .await
7019        {
7020            self.state.note_withheld("fix", &r.withheld);
7021        }
7022        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7023        fix.committed = after != before;
7024        // Judged by what `git` says moved against base, never by the
7025        // fixer's own `addressed`/`rejected` count — see
7026        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7027        // `patch` snapshot above: swallowing this error would default
7028        // `diff_after` to empty, which almost always differs from a
7029        // non-empty `patch` and reads as "progressed" — exactly backwards
7030        // for a `git` failure the stagnation check cannot see through.
7031        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7032        drop(coordination);
7033        let progressed = diff_after != patch;
7034        let commit_note = if fix.committed {
7035            "committed"
7036        } else {
7037            "NO new commit"
7038        };
7039        let tree_note = if progressed {
7040            "changed vs base"
7041        } else {
7042            "unchanged vs base"
7043        };
7044        self.state.event(
7045            "fix",
7046            match &fix.failed {
7047                // Distinct on purpose from "0 addressed, 0 rejected": the
7048                // fixer's own diff still landed (blocking counts do keep
7049                // falling round over round), only its adoption report did
7050                // not come back, so this must never read like every
7051                // finding was reviewed and declined.
7052                Some(reason) => {
7053                    format!(
7054                        "round {round}: fixer's adoption report was lost ({reason}); \
7055                             {commit_note}, tree {tree_note}"
7056                    )
7057                }
7058                None => format!(
7059                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7060                         {tree_note}{}",
7061                    fix.addressed.len(),
7062                    fix.rejected.len(),
7063                    if continuation.outcome == ContinuationOutcome::Resumed {
7064                        format!(
7065                            " (adoption report recovered after {} continuation(s))",
7066                            continuation.attempts
7067                        )
7068                    } else {
7069                        String::new()
7070                    },
7071                ),
7072            },
7073        );
7074        round_record.fix = Some(fix);
7075        round_record.progressed = progressed;
7076        Ok(FixStep::Recorded {
7077            report_lost: matches!(
7078                continuation.outcome,
7079                ContinuationOutcome::Exhausted
7080                    | ContinuationOutcome::QuotaLost
7081                    | ContinuationOutcome::NoSession
7082            ),
7083        })
7084    }
7085
7086    /// Decide, from the last recorded round's own verification, whether
7087    /// stopping the review loop is a hand-off or a genuine block.
7088    ///
7089    /// Called once the loop has given up trying — the round budget is spent,
7090    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7091    /// findings still open, never while a round is still clean or the
7092    /// incomplete-panel case handled inline above. Gate and e2e are facts
7093    /// about the tree; a lingering review finding is an opinion, and this
7094    /// workload's own `magi stats` puts reviewer precision low enough
7095    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7096    /// must not by itself stand between a green, verified change and the
7097    /// human who decides what to do with it. A red e2e is not an opinion, so
7098    /// that case still blocks, with the failing command and a tail of its
7099    /// output recorded here rather than left in `run.json` for someone to go
7100    /// find.
7101    ///
7102    /// A round that deferred its own e2e (see [`Config::graph`]'s
7103    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7104    /// only because nothing ran, and treating an empty list as a passing one
7105    /// here is exactly the "deferred painted green" bug this function exists
7106    /// to not have. When the last round's own verification never resolved —
7107    /// deferred on purpose, or a real attempt the shared build cache blocked
7108    /// — this makes (or retries) the real run, on the actual worktree this
7109    /// loop is about to stop touching, before deciding anything. A
7110    /// resource-blocked attempt is likewise never read as either green or
7111    /// red: it is evidence about the machine, not the patch (see
7112    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7113    /// blocked cache leaves this call without deciding rather than guessing
7114    /// — the caller retries on a later reentry.
7115    /// Record, once, that the review loop handed off over a blocking finding
7116    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7117    /// `land` asks the owner even with `land_approval` off. Called from every
7118    /// path that concludes `Gating`; a reentry keeps the first record.
7119    fn record_contested_handoff(&mut self) {
7120        if self.state.contested_handoff.is_some() {
7121            return;
7122        }
7123        let Some(contested) = self
7124            .state
7125            .reviews
7126            .last()
7127            .and_then(ReviewRound::contested_handoff)
7128        else {
7129            return;
7130        };
7131        self.state.event(
7132            "review",
7133            format!(
7134                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7135                 wait for the owner's approval",
7136                contested.findings.len(),
7137                contested.rejecters.len()
7138            ),
7139        );
7140        self.state.contested_handoff = Some(contested);
7141    }
7142
7143    async fn stop_unresolved_review(
7144        &mut self,
7145        why: &str,
7146        shell: &[String],
7147        worktree: &Path,
7148    ) -> Result<()> {
7149        self.stop_reviewing_inner(why, shell, worktree, true).await
7150    }
7151
7152    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7153        self.stop_reviewing_inner(why, shell, worktree, false).await
7154    }
7155
7156    async fn stop_reviewing_inner(
7157        &mut self,
7158        why: &str,
7159        shell: &[String],
7160        worktree: &Path,
7161        repairable: bool,
7162    ) -> Result<()> {
7163        let round_idx = self.state.reviews.len() - 1;
7164        // A deferred round and a resource-blocked one are the same shape
7165        // here: neither has a real result yet, and both get one more
7166        // attempt. Read off `e2e_status` — the single source for this —
7167        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
7168        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
7169        // still retries instead of being read as a settled result the
7170        // instant it stops being empty.
7171        let needs_catchup_run = matches!(
7172            self.state.reviews[round_idx].e2e_status(),
7173            E2eStatus::Deferred | E2eStatus::ResourceBlocked
7174        );
7175        if needs_catchup_run {
7176            let round = self.state.reviews[round_idx].round;
7177            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7178            let commands = self.state.config.verify.e2e.clone();
7179            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7180            let cache_dir = self.state.config.cache_dir();
7181            let context = format!(
7182                "round {round}: verification unresolved, catching up before the final decision"
7183            );
7184            let wait = self.repository_wait();
7185            let (outcomes, verify_retried) = with_cache_lease(
7186                &mut self.state,
7187                cache_dir.as_deref(),
7188                "e2e",
7189                "e2e",
7190                worktree,
7191                &attempted_head,
7192                timeout,
7193                &context,
7194                &wait,
7195                |state, budget| {
7196                    let shell = shell.to_vec();
7197                    let commands = commands.clone();
7198                    let context = context.clone();
7199                    async move {
7200                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7201                            .await
7202                    }
7203                },
7204            )
7205            .await?;
7206            let last = &mut self.state.reviews[round_idx];
7207            last.e2e = outcomes;
7208            last.verify_retried = verify_retried;
7209            // Always the commit and time this attempt actually targeted,
7210            // whether or not it happens to equal the reviewed `head` and
7211            // whether or not a command finished — see
7212            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7213            // attempt is recorded too, so a later reader sees "attempted
7214            // again at T2" rather than silence.
7215            last.verified_head = Some(attempted_head);
7216            last.verified_at = Some(Timestamp::now());
7217            if verify_inconclusive(&last.e2e) {
7218                // Still not a real result: `e2e_deferred` is left exactly
7219                // as it was, so `needs_catchup_run` above reads
7220                // `ResourceBlocked` (via `e2e_status`, which checks
7221                // `resource_blocked` before `e2e_deferred`) and retries
7222                // again on the next reentry, rather than recording
7223                // contention as a red e2e and blocking the run on it.
7224                self.state.save()?;
7225                return Ok(());
7226            }
7227            last.e2e_deferred = false;
7228        }
7229        let last = &self.state.reviews[round_idx];
7230        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7231
7232        match last.e2e_status() {
7233            E2eStatus::Failed => {
7234                let red: Vec<String> = last
7235                    .e2e
7236                    .iter()
7237                    .filter(|o| !o.ok())
7238                    .map(|o| {
7239                        format!(
7240                            "`{}` -> {:?}\n{}",
7241                            o.command,
7242                            o.code,
7243                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7244                        )
7245                    })
7246                    .collect();
7247                let can_repair = last.blocking == 0 && last.fix.is_none();
7248                self.state.reviews[round_idx].clean = false;
7249                self.state
7250                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7251                self.state.status = RunStatus::Blocked;
7252                if repairable
7253                    && can_repair
7254                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7255                {
7256                    self.state.status = RunStatus::Reviewing;
7257                }
7258            }
7259            // `needs_catchup_run` above already retried once this call; if
7260            // it is still blocked, this is magi's own admission it could
7261            // not get a command to run, never a verdict on the patch — the
7262            // run is left exactly where a later reentry can retry again.
7263            E2eStatus::ResourceBlocked => {
7264                self.state.event(
7265                    "review",
7266                    format!(
7267                        "{why}; e2e could not run (resource unavailable); not \
7268                         deciding yet"
7269                    ),
7270                );
7271            }
7272            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7273                self.state.event(
7274                    "review",
7275                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7276                );
7277                self.record_contested_handoff();
7278                self.state.status = RunStatus::Gating;
7279            }
7280        }
7281        self.state.save()?;
7282        Ok(())
7283    }
7284
7285    // ----------------------------------------------------------------- gate
7286
7287    async fn gate(&mut self) -> Result<()> {
7288        if self.hold_base_sync().await? {
7289            return Ok(());
7290        }
7291        // Judged by the review record itself, not by `status`: a solo
7292        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7293        // reentry (see `judge`), and trusting it here is exactly how a run
7294        // that exhausted its review budget got gated and merged a second
7295        // time around. `review_conclusion` recomputes the review loop's own
7296        // verdict from the round records themselves — `Gating` for a clean
7297        // round or a hand-off (see `stop_reviewing`), anything else means the
7298        // loop is still going or genuinely blocked.
7299        // A base the winner could not be replayed onto is a decision, not a
7300        // round: there is no landing tree to gate. Read as its own record for
7301        // the same reason the review verdict is.
7302        if self.state.status == RunStatus::Failed
7303            || self
7304                .state
7305                .base_sync
7306                .as_ref()
7307                .is_some_and(|s| s.conflict.is_some())
7308            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7309                != Some(RunStatus::Gating)
7310        {
7311            return Ok(());
7312        }
7313        let Some(winner) = self.state.winner().cloned() else {
7314            return Ok(());
7315        };
7316        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7317        self.state.reconcile_verification_head(&head);
7318        self.state.save()?;
7319        let wait = self.repository_wait();
7320        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7321            return Ok(());
7322        }
7323        if self.state.gate_ran {
7324            // `review_loop` derives its conclusion from the clean review
7325            // record on every reentry and therefore puts a completed run back
7326            // in `Gating`. A recorded gate is a stronger, terminal fact:
7327            // retain its original command output (or lack of any, for a repo
7328            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7329            // doc) and restore `Blocked` on a real failure rather than
7330            // pretending the command is still running or running it a second
7331            // time. `gate_ran == false` remains the only shape — unattempted,
7332            // or a resource-blocked retry — that may still need to execute a
7333            // command.
7334            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7335                self.state.status = RunStatus::Blocked;
7336                self.state.save()?;
7337            }
7338            return Ok(());
7339        }
7340        self.state.status = RunStatus::Gating;
7341        let mut outcomes = self.run_gate(&winner).await?;
7342        let mut attempted_head = self.state.gate_head.clone();
7343        loop {
7344            // A resource-blocked outcome means the gate command never actually
7345            // ran - the shared build cache could not be acquired or confirmed
7346            // fresh in time - which is evidence about the machine, not about
7347            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7348            // Recording it as a red gate would mark a run `Blocked` on nothing
7349            // but contention magi has already logged; leaving `self.state.gate`
7350            // empty and `self.state.gate_ran` false instead keeps the shape
7351            // this function already treats as "still needs to run" (see the
7352            // early-return above), so the next call retries the command
7353            // rather than concluding anything.
7354            if verify_inconclusive(&outcomes) {
7355                self.state.save()?;
7356                return Ok(());
7357            }
7358            if outcomes.iter().all(CommandOutcome::ok) {
7359                break;
7360            }
7361            let fixes_before = self.state.gate_fixes.len();
7362            match self.gate_fix_round(&winner, &outcomes).await? {
7363                GateFix::Retry => {
7364                    outcomes = self.run_gate(&winner).await?;
7365                    attempted_head = self.state.gate_head.clone();
7366                }
7367                GateFix::Stop => {
7368                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7369                    if self.state.gate_fixes.len() > fixes_before
7370                        && attempted_head.as_deref() != Some(current.as_str())
7371                        && verify_current_head(&mut self.state, &winner, &wait).await?
7372                    {
7373                        // A new SHA owes a gate even when a spent fixer changed only
7374                        // commit metadata. This does not grant another repair round.
7375                        outcomes = self.run_gate(&winner).await?;
7376                        attempted_head = self.state.gate_head.clone();
7377                    } else {
7378                        break;
7379                    }
7380                }
7381                GateFix::Defer => {
7382                    self.state.save()?;
7383                    return Ok(());
7384                }
7385            }
7386        }
7387        let passed = outcomes.iter().all(CommandOutcome::ok);
7388        self.state.gate = outcomes;
7389        self.state.gate_ran = true;
7390        self.state.gate_head = attempted_head;
7391        self.state.save()?;
7392        if !verify_current_head(&mut self.state, &winner, &wait).await? {
7393            return Ok(());
7394        }
7395        if !passed {
7396            self.state.status = RunStatus::Blocked;
7397            let spent = self.state.gate_fixes.len();
7398            self.state.event(
7399                "gate",
7400                if spent == 0 {
7401                    "gate failed; not merging".to_owned()
7402                } else {
7403                    format!("gate failed after {spent} gate-fix round(s); not merging")
7404                },
7405            );
7406        }
7407        self.state.save()?;
7408        Ok(())
7409    }
7410
7411    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7412    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7413    /// after review is clean and never on a candidate awaiting judging.
7414    ///
7415    /// Never fails the run: a non-zero exit or timeout is a warning and a
7416    /// recorded outcome, and the gate remains the single arbiter. Nothing
7417    /// configured means nothing happens - no event, no commit. `commit_all`
7418    /// commits any leftover change under the neutral identity and returns
7419    /// `false` when the tree is clean, so no empty commit is ever made.
7420    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7421        let commands = state.config.verify.pre_gate.clone();
7422        if commands.is_empty() {
7423            return;
7424        }
7425        let shell = state.config.shell();
7426        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7427        let (outcomes, _) = run_commands(
7428            state,
7429            "pre_gate",
7430            "pre_gate",
7431            0,
7432            &shell,
7433            &commands,
7434            &winner.worktree,
7435            timeout,
7436        )
7437        .await;
7438        for o in &outcomes {
7439            if !o.ok() {
7440                tracing::warn!(
7441                    "pre_gate `{}` failed ({:?}); the gate decides",
7442                    o.command,
7443                    o.code
7444                );
7445            }
7446            state.event(
7447                "pre_gate",
7448                format!(
7449                    "`{}` -> {}",
7450                    o.command,
7451                    if o.ok() {
7452                        "pass".to_owned()
7453                    } else {
7454                        format!(
7455                            "FAIL ({:?})\n{}",
7456                            o.code,
7457                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7458                        )
7459                    }
7460                ),
7461            );
7462        }
7463        state.pre_gate = outcomes;
7464        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7465            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7466                Ok(head) => {
7467                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7468                    state.pre_gate_commit = Some(head);
7469                }
7470                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7471            },
7472            Ok(false) => {}
7473            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7474        }
7475        if let Err(e) = state.save() {
7476            tracing::warn!("could not persist the pre_gate record: {e:#}");
7477        }
7478    }
7479
7480    /// Run `verify.gate` once against the winner's current tree, logging one
7481    /// event per command. Empty when nothing is configured.
7482    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7483        let wait = self.repository_wait();
7484        run_landing_gate(&mut self.state, winner, &wait).await
7485    }
7486
7487    /// One bounded fix round for a failing gate.
7488    ///
7489    /// The fixer is told the failure came from the gate itself, not from a
7490    /// reviewer, and is shown the failed commands, their exit codes and a tail
7491    /// of their output - whatever `[verify].gate` holds, nothing here knows
7492    /// what those commands run. Only a normal non-zero exit that printed
7493    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7494    /// command or a full disk says nothing about the code, and a fixer sent
7495    /// after it can only appease the machine. The round is judged by what git
7496    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7497    /// again before the gate does, so a fix cannot trade a green gate for a
7498    /// red e2e unnoticed.
7499    async fn gate_fix_round(
7500        &mut self,
7501        winner: &Candidate,
7502        outcomes: &[CommandOutcome],
7503    ) -> Result<GateFix> {
7504        let cap = self.state.config.graph.gate_fix_rounds;
7505        let spent = self.state.gate_fixes.len();
7506        if spent >= cap {
7507            if cap > 0 {
7508                self.state.event(
7509                    "gate",
7510                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7511                );
7512            }
7513            return Ok(GateFix::Stop);
7514        }
7515        if !gate_fixable(outcomes) {
7516            self.state.event(
7517                "gate",
7518                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7519                 command or similar); not spending a fix round on it",
7520            );
7521            return Ok(GateFix::Stop);
7522        }
7523        let min_free = self.state.config.disk.min_free_bytes;
7524        if min_free > 0 {
7525            match crate::disk::free_bytes(&winner.worktree) {
7526                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7527                Ok(free) => {
7528                    self.state.event(
7529                        "gate",
7530                        format!(
7531                            "only {free} bytes free ({min_free} required by `[disk] \
7532                             min_free_bytes`); not spending a fix round on a failure the disk \
7533                             may explain"
7534                        ),
7535                    );
7536                    return Ok(GateFix::Stop);
7537                }
7538                Err(e) => {
7539                    self.state.event(
7540                        "gate",
7541                        format!("free disk space could not be measured ({e:#}); no fix round"),
7542                    );
7543                    return Ok(GateFix::Stop);
7544                }
7545            }
7546        }
7547
7548        let attempt = spent + 1;
7549        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7550        let base = self.landing_base();
7551        let fix_prompt = prompt::gate_fix_in(
7552            &self.state.instruction,
7553            &failed,
7554            attempt,
7555            cap,
7556            &self.state.config.graph.language,
7557            &self.state.config.graph.github_language,
7558        );
7559        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7560        let sessions = self.state.config.graph.sessions;
7561        let artifacts = self.merge_repair_artifacts();
7562        self.state.event(
7563            "gate",
7564            format!("gate failed; gate-fix round {attempt} of {cap}"),
7565        );
7566        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7567        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7568        // Persist before invoking the fixer: it may commit and then be
7569        // interrupted before its result (or the following e2e) is saved.
7570        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7571            head: before.clone(),
7572            passed: false,
7573            outcomes: Vec::new(),
7574        });
7575        self.state.save()?;
7576        let (_, seat, out) = self
7577            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7578                prompt: fix_prompt.clone(),
7579                spec,
7580                seat,
7581                cwd: winner.worktree.clone(),
7582                timeout,
7583                allow_write: true,
7584                sessions,
7585                artifacts: artifacts.clone(),
7586                stem: format!("gate-fix-{attempt}"),
7587                handover: None,
7588            })
7589            .await;
7590        // The fixer was never asked: no round is spent (nothing is recorded)
7591        // and a later reentry asks the same agent again.
7592        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7593            return Ok(GateFix::Defer);
7594        }
7595        let mut record = GateFixRecord {
7596            agent: seat.agent.clone(),
7597            failed,
7598            notes: String::new(),
7599            committed: false,
7600            error: None,
7601        };
7602        match out {
7603            AgentOutcome::Ok(o) => {
7604                // A missing report is not a failed fix: the round is judged
7605                // by the tree below, and the report only carries prose.
7606                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7607                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7608                }
7609            }
7610            AgentOutcome::Dropped(_) => {
7611                record.error = Some("the CLI dropped the stream".to_owned());
7612            }
7613            AgentOutcome::Quota(o) => {
7614                self.state.quota.push(QuotaLoss {
7615                    seat: seat.key.clone(),
7616                    node: "gate-fix".to_owned(),
7617                    at: Timestamp::now(),
7618                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7619                });
7620                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7621            }
7622            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7623        }
7624        self.state.seats.insert(seat.key.clone(), seat);
7625        let coordination = self.coordinate(&winner.worktree).await?;
7626        if let Ok(r) = git::rescue_commit(
7627            &winner.worktree,
7628            &format!("magi: gate fix {attempt} (uncommitted work)"),
7629        )
7630        .await
7631        {
7632            self.state.note_withheld("gate-fix", &r.withheld);
7633        }
7634        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7635        record.committed = after != before;
7636        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7637        let note = record.error.clone();
7638        self.state.gate_fixes.push(record);
7639        self.state.save()?;
7640        if !changed {
7641            self.state.event(
7642                "gate",
7643                match note {
7644                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7645                    None => format!("gate-fix round {attempt}: the tree did not change"),
7646                },
7647            );
7648            return Ok(GateFix::Stop);
7649        }
7650        self.state.event(
7651            "gate",
7652            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7653        );
7654
7655        drop(coordination);
7656        let wait = self.repository_wait();
7657        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7658    }
7659
7660    // ---------------------------------------------------------------- merge
7661
7662    /// One read-only rewrite by the summary's author; a text the gate still
7663    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7664    /// applies on their word or on silence. `None` means the run is parked on
7665    /// that question and nothing may be pushed or opened yet.
7666    async fn guarded_pr_message(
7667        &mut self,
7668        winner: &Candidate,
7669        facts: Option<&BranchFacts>,
7670        posting: bool,
7671    ) -> Result<Option<PrMessage>> {
7672        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7673        let head = match final_head.as_deref() {
7674            Some(h) => crate::pr_outcome::FinalHead::At(h),
7675            None => crate::pr_outcome::FinalHead::Unknown,
7676        };
7677        // Only a pull request that is about to be posted pays for a summary.
7678        let summarized = if posting {
7679            Some(self.summarize_change(winner, final_head.as_deref()).await)
7680        } else {
7681            None
7682        };
7683        // A busy shared resource is not a permanent cause: publishing an
7684        // "unavailable" description would stick (a later head move never
7685        // edits the body). Stop before anything is pushed or opened; the
7686        // run is `Blocked` and a resume asks the summary again.
7687        if self.state.resource_block.is_some() {
7688            self.state.event(
7689                "pr-summary",
7690                "a shared resource was busy; nothing was published, resume to summarize again",
7691            );
7692            return Ok(None);
7693        }
7694        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7695            anyhow::bail!(
7696                "the final commit moved during public description generation; retry before publishing"
7697            );
7698        }
7699        let note = match (&summarized, &self.state.change_summary) {
7700            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7701            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7702            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7703            (None, _) => ChangeNote::NotAsked,
7704        };
7705        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7706        if !posting {
7707            let identity = crate::scrub::Identity::current();
7708            return Ok(Some(PrMessage {
7709                title: crate::scrub::scrub(&pr.title, &identity),
7710                body: crate::scrub::scrub(&pr.body, &identity),
7711            }));
7712        }
7713        // This very text was already rejected and asked about: no second
7714        // judge call, rewrite or question, just carry the record on.
7715        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7716        if let Some(known) = self
7717            .state
7718            .github_text
7719            .clone()
7720            .filter(|g| g.fingerprint == raw_fp)
7721        {
7722            return self.settle_withheld(winner, &pr, known).await;
7723        }
7724        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7725        let mut violations =
7726            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7727        let identity = crate::scrub::Identity::current();
7728        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7729            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7730            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7731        {
7732            violations.push(crate::github_text::Violation::SensitiveData);
7733        }
7734        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7735        // only a language violation starts one; sensitive data alone goes
7736        // straight to the owner, with the redacted text as the fallback.
7737        let has_language = violations
7738            .iter()
7739            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7740        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7741            self.state.event(
7742                "github-text",
7743                if has_language {
7744                    format!("description rejected: {violations:?}; requesting one rewrite")
7745                } else {
7746                    format!("description rejected: {violations:?}; asking the owner")
7747                },
7748            );
7749            let mut rewritten = false;
7750            let mut rejected_rewrite = None;
7751            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7752            {
7753                let output = text;
7754                if let Some(title) = summary_title(&output) {
7755                    // Judge the redacted candidate that will actually be posted.
7756                    // Sensitive data alone must not discard a useful rewrite.
7757                    let raw_body = summary_without_title(&output);
7758                    let clean_title = crate::scrub::scrub(&title, &identity);
7759                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7760                    let redacted = clean_title != title || body != raw_body;
7761                    let title = clean_title;
7762                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7763                    if !complete {
7764                        self.state.event(
7765                            "github-text",
7766                            "description rewrite omitted required findings",
7767                        );
7768                    }
7769                    if !body.trim().is_empty() {
7770                        body.push_str("\n\n");
7771                        body.push_str(&public_pr_evidence(&self.state, head));
7772                    }
7773                    let marker = format!("magi:run/{}", self.state.id);
7774                    if !body.trim().is_empty() && !body.contains(&marker) {
7775                        body.push_str(&format!("\n\n{marker}"));
7776                    }
7777                    let rewrite = PrMessage {
7778                        title: title.clone(),
7779                        body: body.clone(),
7780                    };
7781                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7782                    let mut failed = crate::github_text::check_for(
7783                        &self.state.config.graph,
7784                        &title,
7785                        &body,
7786                        redecision,
7787                    );
7788                    if (!crate::github_text::shareable(&title)
7789                        || !crate::github_text::shareable(&body))
7790                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7791                    {
7792                        failed.push(crate::github_text::Violation::SensitiveData);
7793                    }
7794                    if complete
7795                        && !title.trim().is_empty()
7796                        && !body.trim().is_empty()
7797                        && !failed.is_empty()
7798                    {
7799                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7800                            title: title.clone(),
7801                            body: body.clone(),
7802                            failed: failed
7803                                .iter()
7804                                .map(|v| crate::github_text::category(*v).to_owned())
7805                                .collect(),
7806                        });
7807                    }
7808                    if complete
7809                        && !title.trim().is_empty()
7810                        && !raw_body.trim().is_empty()
7811                        && failed.is_empty()
7812                    {
7813                        decision = redecision;
7814                        pr = PrMessage { title, body };
7815                        rewritten = true;
7816                        self.state
7817                            .event("github-text", "description rewrite passed");
7818                        if redacted {
7819                            self.state.event(
7820                                "github-text",
7821                                "sensitive data removed from description rewrite",
7822                            );
7823                        }
7824                    }
7825                }
7826            }
7827            if !rewritten {
7828                self.state.event(
7829                    "github-text",
7830                    "description rewrite unavailable or rejected; asking the owner",
7831                );
7832                if let Some(w) =
7833                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7834                {
7835                    // A different text than the one asked about before: that
7836                    // question no longer describes anything.
7837                    self.retire_github_text_question("the withheld text changed");
7838                    let record = crate::run::GithubTextAsk {
7839                        rewrite: rejected_rewrite,
7840                        fingerprint: raw_fp,
7841                        title: w.title,
7842                        body: w.body,
7843                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7844                        question: None,
7845                        asks: 0,
7846                        resolved: false,
7847                        chosen_title: None,
7848                        choice: None,
7849                    };
7850                    return self.settle_withheld(winner, &pr, record).await;
7851                }
7852            }
7853        }
7854        let (title, body) =
7855            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7856        Ok(Some(PrMessage { title, body }))
7857    }
7858
7859    /// Describe the whole change from its final diff in either run mode.
7860    ///
7861    /// The summary is tied to the final commit and to the merge base of the
7862    /// resolved remote base; a record for any other pair is dropped and the
7863    /// agent asked again, so a description never outlives the diff it was
7864    /// written from. The seat is read-only and the prompt tells it to read the
7865    /// diff and the code around it. `[roles] synthesizer` selects the agents
7866    /// (unset: the default pick), each asked at most once, advancing on an
7867    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7868    /// carries the reason it is unavailable; nothing is invented then.
7869    async fn summarize_change(
7870        &mut self,
7871        winner: &Candidate,
7872        head: Option<&str>,
7873    ) -> std::result::Result<(), String> {
7874        let Some(head) = head else {
7875            return Err("the final commit could not be read".to_owned());
7876        };
7877        let remote = self.state.config.merge.remote.clone();
7878        let tracking = format!("{remote}/{}", self.state.base_branch);
7879        if !git::rev_exists(&winner.worktree, &tracking).await {
7880            return Err(format!("the remote base `{tracking}` could not be read"));
7881        }
7882        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7883            Ok(b) if !b.is_empty() => b,
7884            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7885        };
7886        let evidence = crate::pr_outcome::summary_evidence(
7887            &self.state,
7888            crate::pr_outcome::FinalHead::At(head),
7889        );
7890        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7891        if self
7892            .state
7893            .change_summary
7894            .as_ref()
7895            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7896        {
7897            return Ok(());
7898        }
7899        if self.state.change_summary.take().is_some() {
7900            self.state.event(
7901                "pr-summary",
7902                "the final diff or evidence changed; the earlier change summary was discarded",
7903            );
7904        }
7905        // Save the source snapshot even if no generation agent is available.
7906        if let Err(e) = write_artifact(
7907            &self.state,
7908            &format!("public-description-{fingerprint}.md"),
7909            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7910        ) {
7911            tracing::warn!("could not save public description evidence: {e:#}");
7912        }
7913        let chain = match agent::pick_chain(
7914            &self.state.config.agents,
7915            self.state.config.roles.synthesizer.as_ref(),
7916            &agent::installed,
7917            "synthesizer",
7918        ) {
7919            Ok(chain) => chain,
7920            Err(e) => {
7921                tracing::warn!("no agent for the change summary: {e:#}");
7922                self.state.event("pr-summary", "no summary agent available");
7923                return Err("no agent was available to summarize it".to_owned());
7924            }
7925        };
7926        let subjects = git::subjects(&winner.worktree, &base, head)
7927            .await
7928            .unwrap_or_default();
7929        let stat = git::diff_stat(&winner.worktree, &base, head)
7930            .await
7931            .unwrap_or_default();
7932        let stat: String = stat.chars().take(6_000).collect();
7933        let mut prompt = crate::prompt::change_summary_in(
7934            &base,
7935            head,
7936            &subjects,
7937            stat.trim(),
7938            &self.state.config.graph.github_language,
7939        );
7940        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7941        let sessions = self.state.config.graph.sessions;
7942        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7943        let prompts = self.state.config.prompts.clone();
7944        let cache = self.state.config.cache_dir();
7945        let run = self.state.id.clone();
7946        for (i, spec) in chain.into_iter().enumerate() {
7947            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7948            let github_spec = match agent::for_github_text(
7949                &spec,
7950                &self.state.config.graph.github_language,
7951                &winner.worktree,
7952            ) {
7953                Ok(spec) => spec,
7954                Err(e) => {
7955                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7956                    continue;
7957                }
7958            };
7959            let job = SeatJob {
7960                spec: github_spec,
7961                seat,
7962                cwd: winner.worktree.clone(),
7963                prompt: prompt.clone(),
7964                timeout,
7965                allow_write: false,
7966                sessions,
7967                artifacts: agent::artifacts_dir(&self.state.dir()),
7968                stem: if i == 0 {
7969                    "change-summary".to_owned()
7970                } else {
7971                    format!("change-summary-{}", spec.id)
7972                },
7973                handover: None,
7974            };
7975            let ctx = WaveCtx {
7976                run: &run,
7977                node: "pr-summary",
7978                prompts: &prompts,
7979                cache: cache.as_deref(),
7980                round: None,
7981                carry_seats: false,
7982                pauses: Some((&self.pause, &self.interrupt)),
7983            };
7984            let (seat, outcome) =
7985                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7986            self.state.seats.insert(seat.key.clone(), seat);
7987            let text = match outcome {
7988                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7989                AgentOutcome::ResourceBlocked(why) => {
7990                    // No agent was asked: the run stops resumably before
7991                    // anything is published (see `guarded_pr_message`).
7992                    if self.state.resource_block.is_none() {
7993                        self.state.resource_block = Some(crate::run::ResourceBlock {
7994                            cause: crate::run::ResourceBlockCause::Other,
7995                            node: "pr-summary".to_owned(),
7996                            kind: "shared resource".to_owned(),
7997                            detail: why,
7998                            at: Timestamp::now(),
7999                        });
8000                    }
8001                    return Err("a shared resource was busy".to_owned());
8002                }
8003                _ => continue,
8004            };
8005            let blind = &self.state.config.blind;
8006            let Some(title) = summary_title(&text) else {
8007                continue;
8008            };
8009            let body: String = summary_without_title(&text)
8010                .lines()
8011                .filter(|l| !l.contains("magi:run/"))
8012                .collect::<Vec<_>>()
8013                .join("\n");
8014            let body = blind::sanitize_prose(body.trim(), blind);
8015            let title = blind::sanitize_prose(&title, blind);
8016            if body.trim().is_empty() || title.trim().is_empty() {
8017                continue;
8018            }
8019            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8020                continue;
8021            }
8022            // The branch must still be the one the agent was shown.
8023            if git::rev_parse(&winner.worktree, "HEAD")
8024                .await
8025                .ok()
8026                .as_deref()
8027                != Some(head)
8028            {
8029                return Err("the branch moved while the summary was written".to_owned());
8030            }
8031            if git::merge_base(&winner.worktree, &tracking, head)
8032                .await
8033                .ok()
8034                .as_deref()
8035                != Some(base.as_str())
8036                || crate::pr_outcome::summary_evidence(
8037                    &self.state,
8038                    crate::pr_outcome::FinalHead::At(head),
8039                ) != evidence
8040            {
8041                return Err("the summary evidence changed while it was written".to_owned());
8042            }
8043            self.state.change_summary = Some(crate::run::ChangeSummary {
8044                head: head.to_owned(),
8045                base,
8046                evidence: fingerprint,
8047                title,
8048                body,
8049            });
8050            self.state.event(
8051                "pr-summary",
8052                format!("change summary written by {} for the final diff", spec.id),
8053            );
8054            if let Err(e) = self.state.save() {
8055                tracing::warn!("could not save the change summary: {e:#}");
8056            }
8057            return Ok(());
8058        }
8059        self.state
8060            .event("pr-summary", "no agent produced a change summary");
8061        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8062    }
8063
8064    /// One read-only rewrite request for a withheld description. The
8065    /// implementer's own conversation is tried first when it exists and can be
8066    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8067    /// writes it in a separate conversation, skipping the implementer's agent.
8068    /// The prompt carries the whole text, so no agent needs any memory. Each
8069    /// agent is asked at most once and the chain advances on an error, quota,
8070    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8071    /// that has run out of quota must not end the rewrite. Nothing is recorded
8072    /// as a handover.
8073    async fn rewrite_pr_text(
8074        &mut self,
8075        winner: &Candidate,
8076        pr: &PrMessage,
8077        violations: &[crate::github_text::Violation],
8078    ) -> Option<String> {
8079        let prompt = format!(
8080            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8081            pr.title, pr.body
8082        );
8083        let sessions = self.state.config.graph.sessions;
8084        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8085        // (spec, the resumed implementer's seat). A fallback seat is created
8086        // only when its turn comes, so an unused fallback leaves no record.
8087        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8088        let key = format!("impl-{}", winner.label);
8089        if let Some(spec) = self
8090            .state
8091            .config
8092            .agents
8093            .iter()
8094            .find(|a| a.id == winner.agent)
8095            .cloned()
8096            && let Some(existing) = self.state.seats.get(&key)
8097            && existing.agent == spec.id
8098            && agent::has_session(spec.kind, existing, sessions)
8099        {
8100            attempts.push((spec, Some(existing.clone())));
8101        }
8102        match agent::pick_chain(
8103            &self.state.config.agents,
8104            self.state.config.roles.language_judge.as_ref(),
8105            &agent::installed,
8106            "language_judge",
8107        ) {
8108            Ok(chain) => {
8109                for spec in chain {
8110                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8111                        continue;
8112                    }
8113                    attempts.push((spec, None));
8114                }
8115            }
8116            Err(e) if attempts.is_empty() => {
8117                tracing::warn!("no agent for the description rewrite: {e:#}");
8118                self.state
8119                    .event("github-text", "no rewrite agent available");
8120                return None;
8121            }
8122            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8123        }
8124        let prompts = self.state.config.prompts.clone();
8125        let cache = self.state.config.cache_dir();
8126        let run = self.state.id.clone();
8127        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8128            let resumed = resumed_seat.is_some();
8129            let seat = resumed_seat.unwrap_or_else(|| {
8130                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8131            });
8132            let github_spec = match agent::for_github_text(
8133                &spec,
8134                &self.state.config.graph.github_language,
8135                &winner.worktree,
8136            ) {
8137                Ok(spec) => spec,
8138                Err(e) => {
8139                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8140                    continue;
8141                }
8142            };
8143            let job = SeatJob {
8144                spec: github_spec,
8145                seat,
8146                cwd: winner.worktree.clone(),
8147                prompt: prompt.clone(),
8148                timeout: retry_budget(timeout_full, resumed),
8149                allow_write: false,
8150                sessions,
8151                artifacts: agent::artifacts_dir(&self.state.dir()),
8152                stem: if i == 0 {
8153                    "github-text-rewrite".to_owned()
8154                } else {
8155                    format!("github-text-rewrite-{}", spec.id)
8156                },
8157                handover: None,
8158            };
8159            let ctx = WaveCtx {
8160                run: &run,
8161                node: "github-text",
8162                prompts: &prompts,
8163                cache: cache.as_deref(),
8164                round: None,
8165                carry_seats: false,
8166                pauses: Some((&self.pause, &self.interrupt)),
8167            };
8168            let (seat, outcome) =
8169                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8170            self.state.seats.insert(seat.key.clone(), seat);
8171            match outcome {
8172                AgentOutcome::Ok(o)
8173                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8174                {
8175                    return Some(o.text);
8176                }
8177                AgentOutcome::ResourceBlocked(_) => return None,
8178                _ => {}
8179            }
8180        }
8181        None
8182    }
8183
8184    /// Abandon the standing posting-gate question, if any.
8185    fn retire_github_text_question(&mut self, why: &str) {
8186        let Some(id) = self
8187            .state
8188            .github_text
8189            .as_ref()
8190            .and_then(|g| g.question.clone())
8191        else {
8192            return;
8193        };
8194        let _ = ask::Questions::open().update(&id, |q| {
8195            q.abandon(why);
8196            Ok(())
8197        });
8198    }
8199
8200    /// Carry a withheld text to its end: file the question, park while it is
8201    /// open, and on an answer, silence or timeout pick the text to post. The
8202    /// decision is saved before it is used, so it applies once.
8203    async fn settle_withheld(
8204        &mut self,
8205        winner: &Candidate,
8206        pr: &PrMessage,
8207        mut g: crate::run::GithubTextAsk,
8208    ) -> Result<Option<PrMessage>> {
8209        use crate::github_text as gt;
8210        let store = ask::Questions::open();
8211        let timeout = self.state.config.graph.answer_timeout;
8212        if !g.resolved {
8213            let mut standing = g
8214                .question
8215                .as_ref()
8216                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8217            if let Some(q) = standing.as_ref()
8218                && q.status.open()
8219                && gt::expired(q, timeout)
8220            {
8221                let why = format!("no answer within {}s of asking", timeout.max(1));
8222                standing = store
8223                    .update(&q.id, |q| {
8224                        q.abandon(&why);
8225                        Ok(())
8226                    })
8227                    .ok()
8228                    .map(|(q, ())| q);
8229            }
8230            let mut ask_again: Option<Option<Vec<String>>> = None;
8231            match standing {
8232                // Recorded but never written (a stop between the two saves),
8233                // or removed: ask again without spending the retry.
8234                None if g.asks <= 1 => {
8235                    g.asks = 0;
8236                    ask_again = Some(None);
8237                }
8238                None => {
8239                    g.resolved = true;
8240                    g.chosen_title = None;
8241                    g.choice = Some(GithubTextChoice::Fallback);
8242                }
8243                Some(q) if q.status.open() => {
8244                    self.park_on_github_text(&g)?;
8245                    return Ok(None);
8246                }
8247                Some(q) => match gt::read_reply(&q) {
8248                    gt::Reply::Fallback => {
8249                        g.resolved = true;
8250                        g.chosen_title = None;
8251                        g.choice = Some(GithubTextChoice::Fallback);
8252                    }
8253                    gt::Reply::Original => {
8254                        g.resolved = true;
8255                        g.chosen_title = None;
8256                        g.choice = Some(GithubTextChoice::Original);
8257                    }
8258                    gt::Reply::Rewrite => {
8259                        g.resolved = true;
8260                        g.chosen_title = None;
8261                        g.choice = Some(if g.rewrite.is_some() {
8262                            GithubTextChoice::Rewrite
8263                        } else {
8264                            GithubTextChoice::Fallback
8265                        });
8266                    }
8267                    gt::Reply::Title(text) => {
8268                        let first = text
8269                            .lines()
8270                            .map(str::trim)
8271                            .find(|l| !l.is_empty())
8272                            .unwrap_or("");
8273                        let decision = if first.is_empty() {
8274                            None
8275                        } else {
8276                            let candidate = PrMessage {
8277                                title: first.to_owned(),
8278                                body: String::new(),
8279                            };
8280                            self.judge_pr_language(&winner.worktree, &candidate).await
8281                        };
8282                        match gt::vet_title(
8283                            &text,
8284                            decision,
8285                            self.state.config.graph.github_english(),
8286                        ) {
8287                            Ok(title) => {
8288                                g.resolved = true;
8289                                g.chosen_title = Some(title);
8290                                g.choice = Some(GithubTextChoice::MyTitle);
8291                            }
8292                            Err(cats) if g.asks < 2 => {
8293                                ask_again = Some(Some(
8294                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8295                                ))
8296                            }
8297                            Err(_) => {
8298                                g.resolved = true;
8299                                g.chosen_title = None;
8300                                g.choice = Some(GithubTextChoice::Fallback);
8301                            }
8302                        }
8303                    }
8304                },
8305            }
8306            if let Some(rejected) = ask_again {
8307                // A retry (after a rejected replacement title) is still about
8308                // the original text: the same ORIGINAL / FALLBACK comparison.
8309                let w = gt::Withheld {
8310                    title: g.title,
8311                    body: g.body,
8312                    categories: g.categories.clone(),
8313                };
8314                // Named by the original text's fingerprint alone, so a resume
8315                // rewrites the same file and the detail stays identical.
8316                let artifact = match crate::run::write_artifact(
8317                    &self.state,
8318                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8319                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8320                ) {
8321                    Ok(p) => Some(p),
8322                    Err(e) => {
8323                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8324                        None
8325                    }
8326                };
8327                let (detail, comparison) = gt::question_content(
8328                    &w,
8329                    &pr.title,
8330                    &pr.body,
8331                    &self.state.id,
8332                    rejected.as_deref(),
8333                    artifact.as_deref(),
8334                    g.rewrite.as_ref(),
8335                );
8336                let mut q = ask::Question::new(
8337                    self.state.id.clone(),
8338                    gt::ASK_NODE.to_owned(),
8339                    gt::ASK_SEAT.to_owned(),
8340                    gt::question_summary(&self.state.config.graph.language, &w),
8341                    detail,
8342                    gt::question_choices(&w, g.rewrite.as_ref()),
8343                );
8344                q.posting_comparison = Some(comparison);
8345                q.answer_timeout = timeout;
8346                // Recorded before it is written: a stop in between leaves a
8347                // record naming a question the resume then files again.
8348                g.question = Some(q.id.clone());
8349                g.asks += 1;
8350                self.state.github_text = Some(g.clone());
8351                self.state.event(
8352                    "github-text",
8353                    format!("asking the owner about the withheld text ({})", q.short()),
8354                );
8355                self.state.save()?;
8356                store
8357                    .put_executable(&mut q)
8358                    .context("file the posting-gate question")?;
8359                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8360                    tracing::warn!(
8361                        "could not notify about posting-gate question {}: {e:#}",
8362                        q.short()
8363                    );
8364                }
8365                self.park_on_github_text(&g)?;
8366                return Ok(None);
8367            }
8368        }
8369        let resolved = g.chosen_title.clone();
8370        let original = g.choice == Some(GithubTextChoice::Original);
8371        // Field names only: the event log never records the text.
8372        self.state.event(
8373            "github-text",
8374            if original {
8375                format!(
8376                    "posting the original text for: {}",
8377                    match (g.title, g.body) {
8378                        (true, true) => "title, description",
8379                        (true, false) => "title",
8380                        _ => "description",
8381                    }
8382                )
8383            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8384                "posting the saved rewrite title and description".to_owned()
8385            } else if resolved.is_some() {
8386                "posting the owner's replacement title".to_owned()
8387            } else {
8388                "posting the neutral text for the withheld fields".to_owned()
8389            },
8390        );
8391        self.state.github_text = Some(g.clone());
8392        self.state.save()?;
8393        if g.choice == Some(GithubTextChoice::Rewrite)
8394            && let Some(r) = g.rewrite.as_ref()
8395        {
8396            return Ok(Some(PrMessage {
8397                title: r.title.clone(),
8398                body: r.body.clone(),
8399            }));
8400        }
8401        let w = gt::Withheld {
8402            title: g.title,
8403            body: g.body,
8404            categories: g.categories.clone(),
8405        };
8406        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8407        if let Some(t) = resolved {
8408            title = t;
8409        }
8410        // Every withheld field is now neutral or vetted English.
8411        let english = gt::LanguageDecision {
8412            title_english: true,
8413            body_english: true,
8414            body_complete: true,
8415        };
8416        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8417        if original {
8418            // The owner's explicit choice overrides the gate for the fields it
8419            // was made for; the others keep what the gate produced.
8420            if g.title {
8421                title = pr.title.clone();
8422            }
8423            if g.body {
8424                body = pr.body.clone();
8425            }
8426        }
8427        Ok(Some(PrMessage { title, body }))
8428    }
8429
8430    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8431        self.state.github_text = Some(g.clone());
8432        self.state.end_pass_parked(ParkCause::Question);
8433        self.state.parked = true;
8434        self.state.event(
8435            "github-text",
8436            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8437        );
8438        self.state.save()
8439    }
8440
8441    /// Ask `[roles] language_judge` about this text and record which source
8442    /// decided; the text itself is never recorded.
8443    async fn judge_pr_language(
8444        &mut self,
8445        cwd: &Path,
8446        pr: &PrMessage,
8447    ) -> Option<crate::github_text::LanguageDecision> {
8448        self.state.config.roles.language_judge.as_ref()?;
8449        if !self.state.config.graph.language_gate() {
8450            return None;
8451        }
8452        let decision =
8453            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8454        self.state.event(
8455            "github-text",
8456            if decision.is_some() {
8457                "language decided by the language judge"
8458            } else {
8459                "language judge unavailable; using built-in heuristics"
8460            },
8461        );
8462        decision
8463    }
8464
8465    async fn merge(&mut self) -> Result<()> {
8466        if self.resource_halt()? || self.hold_base_sync().await? {
8467            return Ok(());
8468        }
8469        // Same reasoning as `gate`: ask the review and gate records directly
8470        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8471        // skip can rewrite on reentry to something that no longer says
8472        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8473        // so a hand-off (open findings, green verification) reaches merge
8474        // exactly like a genuinely clean round does.
8475        //
8476        // A run resumed mid-`land` never reaches here at all: `execute`
8477        // recognises `RunStatus::Landing` before it even calls `prep`, and
8478        // routes straight to `run_land` instead. That has to happen a level
8479        // up from this function, not with a check in here, because
8480        // `review_loop`'s own status recomputation (see its doc) runs
8481        // *before* `merge` on every reentry and would otherwise overwrite
8482        // the `Landing` marker with `Gating` before this node ever saw it.
8483        if self
8484            .state
8485            .base_sync
8486            .as_ref()
8487            .is_some_and(|s| s.conflict.is_some())
8488            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8489                != Some(RunStatus::Gating)
8490            // `gate_ran == false` is not "passed" - `gate` leaves it false
8491            // both before it has ever run and when its last attempt was
8492            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8493            // permission to merge on nothing but the review record. Only a
8494            // gate that actually ran may proceed; an intentional local
8495            // review can also finish with verification explicitly not configured.
8496            // `RunState::gate_status` computes that distinction.
8497            || !self.state.gate_allows_progress()
8498        {
8499            return Ok(());
8500        }
8501        // This node's own record, not `status`: `status == Ready` is not
8502        // unique to the harmless `MergeMode::None` path this line was
8503        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8504        // run's PR was closed without merging — and on that run `mode` is
8505        // still `Pr`, so a reentry that fell through here would push and
8506        // open a second pull request. `self.state.merge` is set exactly once
8507        // this node (or `land`) has already produced a verdict, under every
8508        // mode, which is what "already done" actually means here.
8509        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8510            return Ok(());
8511        }
8512        let Some(winner) = self.state.winner().cloned() else {
8513            return Ok(());
8514        };
8515        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8516        if self.state.gate_head.as_deref() != Some(head.as_str()) {
8517            self.state.status = RunStatus::Blocked;
8518            self.state.event(
8519                "gate",
8520                "winner changed after gate; fresh verification required",
8521            );
8522            self.state.save()?;
8523            return Ok(());
8524        }
8525        let wait = self.repository_wait();
8526        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8527            return Ok(());
8528        }
8529        let repo = self.state.repo.clone();
8530        let base = self.state.base_branch.clone();
8531        let mode = self.state.config.merge.mode;
8532        let style = self.state.config.merge.style;
8533        let facts = if is_review_run(&self.state) {
8534            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8535            let start = review_base(
8536                &repo,
8537                &self.state.config.merge.remote,
8538                &base,
8539                &self.state.base_commit,
8540                &winner.branch,
8541            )
8542            .await;
8543            branch_facts(&repo, &start, &winner.branch).await
8544        } else {
8545            None
8546        };
8547        // `None` when the base could not be freshly read: then an adopted
8548        // pull request's title is left alone.
8549        let leaked = if is_review_run(&self.state) {
8550            leaked_subjects(&self.state, &winner.branch).await
8551        } else {
8552            Some(Vec::new())
8553        };
8554        // `None`: parked on the owner's word about a withheld title or body.
8555        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8556        // the resume arrives here again.
8557        let Some(pr) = self
8558            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8559            .await?
8560        else {
8561            return Ok(());
8562        };
8563        let message = pr.commit_message();
8564
8565        let outcome = match mode {
8566            MergeMode::None => MergeOutcome {
8567                mode,
8568                ok: true,
8569                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8570                empty: false,
8571            },
8572            MergeMode::Pr | MergeMode::Local
8573                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8574            {
8575                MergeOutcome {
8576                    mode,
8577                    ok: false,
8578                    detail: empty_candidate_detail(&self.state, &base),
8579                    empty: true,
8580                }
8581            }
8582            MergeMode::Local => {
8583                let on = git::current_branch(&repo).await?;
8584                if on.as_deref() != Some(base.as_str()) {
8585                    MergeOutcome {
8586                        mode,
8587                        ok: false,
8588                        detail: format!(
8589                            "{} has {} checked out, not the base branch {base}",
8590                            repo.display(),
8591                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8592                        ),
8593                        empty: false,
8594                    }
8595                } else if !git::is_clean(&repo).await? {
8596                    MergeOutcome {
8597                        mode,
8598                        ok: false,
8599                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8600                        empty: false,
8601                    }
8602                } else {
8603                    let out = match style {
8604                        MergeStyle::Merge => {
8605                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8606                        }
8607                        MergeStyle::Squash => {
8608                            git::merge_squash(&repo, &winner.branch, &message).await?
8609                        }
8610                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8611                    };
8612                    MergeOutcome {
8613                        mode,
8614                        ok: out.ok(),
8615                        detail: if out.ok() { out.stdout } else { out.stderr },
8616                        empty: false,
8617                    }
8618                }
8619            }
8620            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8621                self.state.pr_publication_pending = true;
8622                self.state.save()?;
8623                let out =
8624                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8625                self.state.pr_publication_pending = out.is_err();
8626                MergeOutcome {
8627                    mode,
8628                    ok: out.is_ok(),
8629                    detail: match out {
8630                        Ok(url) => url,
8631                        Err(e) => format!("{e:#}"),
8632                    },
8633                    empty: false,
8634                }
8635            }
8636            MergeMode::Pr => {
8637                let remote = self.state.config.merge.remote.clone();
8638                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8639                if !pushed.ok() {
8640                    MergeOutcome {
8641                        mode,
8642                        ok: false,
8643                        detail: pushed.stderr,
8644                        empty: false,
8645                    }
8646                } else {
8647                    // A retry or resume of a run whose branch already has an
8648                    // open pull request adopts it rather than failing on a
8649                    // duplicate. Only this winner branch into this base:
8650                    // `branch_for` derives the name from the run id, so a
8651                    // different run's pull request never matches.
8652                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8653                    let out = match pr_merge_plan(found) {
8654                        PrPlan::Create => {
8655                            gh_pr_create(
8656                                &winner.worktree,
8657                                &base,
8658                                &winner.branch,
8659                                &pr.title,
8660                                &pr.body,
8661                            )
8662                            .await
8663                        }
8664                        PrPlan::Adopt { url, title } => {
8665                            self.state
8666                                .event("merge", format!("Pr: adopted open pull request {url}"));
8667                            if self.state.github_text.as_ref().is_some_and(|g| {
8668                                g.resolved
8669                                    && g.choice == Some(GithubTextChoice::Rewrite)
8670                                    && g.rewrite.is_some()
8671                            }) {
8672                                // The owner chose the entire pair, even if the
8673                                // existing title already matches or was edited.
8674                                land::set_pr_text_verbatim(
8675                                    &winner.worktree,
8676                                    &url,
8677                                    &pr.title,
8678                                    &pr.body,
8679                                )
8680                                .await
8681                                .map(|()| url)
8682                            } else {
8683                                if title != pr.title
8684                                    && (!is_review_run(&self.state)
8685                                        || leaked
8686                                            .as_deref()
8687                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8688                                    && let Err(e) =
8689                                        if self.state.github_text.as_ref().is_some_and(|g| {
8690                                            g.resolved
8691                                                && g.title
8692                                                && g.choice == Some(GithubTextChoice::Original)
8693                                        }) {
8694                                            // The owner chose to post this title unmodified.
8695                                            land::set_pr_title_verbatim(
8696                                                &winner.worktree,
8697                                                &url,
8698                                                &pr.title,
8699                                            )
8700                                            .await
8701                                        } else {
8702                                            land::set_pr_title(
8703                                                &mut self.state,
8704                                                &winner.worktree,
8705                                                &url,
8706                                                &pr.title,
8707                                            )
8708                                            .await
8709                                        }
8710                                {
8711                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8712                                    self.state
8713                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8714                                }
8715                                Ok(url)
8716                            }
8717                        }
8718                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8719                    };
8720                    match out {
8721                        Ok(url) => MergeOutcome {
8722                            mode,
8723                            ok: true,
8724                            detail: url,
8725                            empty: false,
8726                        },
8727                        Err(e) => MergeOutcome {
8728                            mode,
8729                            ok: false,
8730                            detail: e.to_string(),
8731                            empty: false,
8732                        },
8733                    }
8734                }
8735            }
8736        };
8737
8738        self.state.status = match (mode, outcome.ok) {
8739            (MergeMode::None, _) => RunStatus::Ready,
8740            (_, true) if outcome.empty => RunStatus::Ready,
8741            (MergeMode::Pr, true) => RunStatus::Ready,
8742            (_, true) => RunStatus::Merged,
8743            (_, false) => RunStatus::Blocked,
8744        };
8745        self.state.event(
8746            "merge",
8747            format!(
8748                "{:?}: {}",
8749                mode,
8750                outcome.detail.lines().next().unwrap_or("")
8751            ),
8752        );
8753        self.state.merge = Some(outcome);
8754        self.state.save()?;
8755
8756        // The PR is open and the run would historically stop here, leaving the
8757        // operator to watch checks, feed review comments back to a fixer, and
8758        // merge. That was done by hand six times in one session before this
8759        // existed. Opt-in, because merging is the one irreversible thing magi
8760        // can do to a repository.
8761        if self.state.config.graph.land
8762            && mode == MergeMode::Pr
8763            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8764        {
8765            self.run_land().await?;
8766        }
8767        // `run_land` may have left `status` at `Landing` - still waiting on
8768        // CI or the owner's approval, not actually settled - so this has to
8769        // read whatever `status` ended up as here, not the `Merged` this
8770        // function set a few lines up.
8771        self.settle_questions();
8772        Ok(())
8773    }
8774
8775    /// Enter `land`.
8776    ///
8777    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8778    /// a resumed run's re-entry. `land::land` itself is what serialises the
8779    /// two git-mutating moments inside the loop — the rebase push and
8780    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8781    /// to hold a lock across the whole call, and doing so would serialise
8782    /// this run's CI wait against a *different* run's land-approval resume
8783    /// in the same repository, which is exactly the "must not wait on
8784    /// another task" property the daemon's slot-freeing exists to give.
8785    async fn run_land(&mut self) -> Result<()> {
8786        crate::pr_head::ensure_owner(&self.state)?;
8787        let url = self
8788            .state
8789            .pr
8790            .as_ref()
8791            .map(|pr| pr.url.clone())
8792            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8793            .unwrap_or_default();
8794        let url = url.lines().next().unwrap_or("").trim().to_owned();
8795        if !url.starts_with("http") {
8796            return Ok(());
8797        }
8798        // A land failure is not a lost run: the work is on a branch and the
8799        // pull request is open, which is exactly where a human takes over.
8800        loop {
8801            match land::land(&mut self.state, &url).await {
8802                Ok(_)
8803                    if self
8804                        .state
8805                        .merge_repairs
8806                        .last()
8807                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8808                {
8809                    if self.repair_merge_review().await? {
8810                        continue;
8811                    }
8812                }
8813                Ok(pr) if self.state.parked => {
8814                    // `land` already saved the parked marker; nothing here
8815                    // overrides `status` back to a terminal value while an
8816                    // approval is still outstanding.
8817                    let _ = pr;
8818                }
8819                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8820                    self.state.status = RunStatus::Superseded;
8821                    self.state.save()?;
8822                }
8823                Ok(pr) => {
8824                    self.state.status = match pr.state {
8825                        land::PrLifecycle::Merged => RunStatus::Merged,
8826                        _ => RunStatus::Blocked,
8827                    };
8828                    // Downstream of a confirmed merge only - see
8829                    // `bump::should_release_bump`'s own doc for why this one
8830                    // check covers all three of `land`'s success paths.
8831                    // Best-effort: the run already landed, so a failure here
8832                    // (the decision call, `gh`, `cargo`) is recorded and never
8833                    // turns a landed run into a failed one.
8834                    if bump::should_release_bump(self.state.status)
8835                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8836                    {
8837                        // The event is the run's own record. Not-eligible cases
8838                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8839                        // `Err` is a bump that was tried and failed:
8840                        // `after_merge` itself raises the operator notice for
8841                        // that, whether or not a release PR exists yet.
8842                        self.state
8843                            .event("bump", format!("release bump skipped: {e:#}"));
8844                    }
8845                    // Independent of the bump, and best-effort in the same way:
8846                    // findings the merge left open become follow-up tasks.
8847                    if self.state.status == RunStatus::Merged {
8848                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8849                    }
8850                    self.state.save()?;
8851                }
8852                Err(e) => {
8853                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8854                        return Err(e);
8855                    }
8856                    self.state.status = RunStatus::Blocked;
8857                    self.state.event("land", format!("gave up: {e}"));
8858                    self.state.save()?;
8859                }
8860            }
8861            break;
8862        }
8863        Ok(())
8864    }
8865
8866    /// One repair cycle stays under the existing driver/task claim. An
8867    /// interrupted fixer is rolled back and held rather than run twice.
8868    async fn repair_merge_review(&mut self) -> Result<bool> {
8869        use crate::run::MergeRepairPhase as Phase;
8870        crate::pr_head::ensure_owner(&self.state)?;
8871        let i = self.state.merge_repairs.len() - 1;
8872        let request = self.state.merge_repairs[i].clone();
8873        if request.phase == Phase::Fixing {
8874            let restored = match self.state.winner().cloned() {
8875                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8876                None => Err(anyhow::anyhow!("repair has no winning branch")),
8877            };
8878            let why = match restored {
8879                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8880                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8881                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8882            };
8883            self.hold_merge_repair(i, why)?;
8884            return Ok(false);
8885        }
8886        // A completed repair gate is a publication checkpoint. On reentry,
8887        // reject external changes before the ordinary gate path can replace
8888        // that evidence by verifying a new head. Requested repairs still
8889        // start their own review generation below.
8890        if request.phase == Phase::Reviewing && self.state.gate_ran {
8891            let winner = self
8892                .state
8893                .winner()
8894                .context("repair has no winning branch")?;
8895            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8896            if self.state.gate_head.as_deref() != Some(head.as_str())
8897                || !git::is_clean(&winner.worktree).await?
8898            {
8899                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8900                return Ok(false);
8901            }
8902        }
8903        let failure = if i >= self.state.config.graph.land_rounds
8904            || self.state.config.graph.review_rounds == 0
8905        {
8906            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8907        } else {
8908            None
8909        };
8910        if let Some(why) = failure {
8911            self.hold_merge_repair(i, why)?;
8912            return Ok(false);
8913        }
8914        if request.phase == Phase::Requested {
8915            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8916                Ok(pr) => pr,
8917                Err(e) => {
8918                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8919                    return Ok(false);
8920                }
8921            };
8922            self.state.merge_repairs[i].phase = Phase::Fixing;
8923            self.state.land_approval = None;
8924            self.state.land_armed_head = None;
8925            self.state.save()?;
8926            // Serialize the exact records, including finding ids/details,
8927            // reconsideration reasons, final votes and verification output.
8928            let evidence = serde_json::to_string_pretty(&request)?;
8929            let reason = format!(
8930                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8931            );
8932            let budget = self.state.config.graph.land_rounds;
8933            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8934            match result {
8935                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8936                Ok(land::Fixed::Failed(why)) => {
8937                    self.hold_merge_repair(
8938                        i,
8939                        format!(
8940                            "repair failed: {why}; inspect the branch and run an explicit review"
8941                        ),
8942                    )?;
8943                    return Ok(false);
8944                }
8945                Err(e) => {
8946                    self.hold_merge_repair(
8947                        i,
8948                        format!(
8949                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8950                        ),
8951                    )?;
8952                    return Ok(false);
8953                }
8954            }
8955            // Even no change gets a real new reviewer wave. The previous
8956            // evidence remains in the request; a former clean/concluded round
8957            // cannot short-circuit the new review or gate.
8958            self.begin_merge_repair_review(i)?;
8959        }
8960        self.review_loop().await?;
8961        self.gate().await?;
8962        if self.resource_halt()? || self.state.parked {
8963            return Ok(false);
8964        }
8965        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8966            != Some(RunStatus::Gating)
8967            || !self.state.gate_allows_progress()
8968        {
8969            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8970            return Ok(false);
8971        }
8972        // Review and gate may have added commits after the initial repair.
8973        // Publish exactly their final head to the existing PR, with a lease.
8974        let winner = self
8975            .state
8976            .winner()
8977            .cloned()
8978            .context("repair has no winning branch")?;
8979        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8980        if self.state.gate_head.as_deref() != Some(after.as_str())
8981            || !git::is_clean(&winner.worktree).await?
8982        {
8983            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8984            return Ok(false);
8985        }
8986        let expected = self
8987            .state
8988            .pr_pushed_head
8989            .as_deref()
8990            .unwrap_or(&request.head)
8991            .to_owned();
8992        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8993        // Push the checked commit itself: a local ref moving after the check
8994        // must not change which commit the forge receives.
8995        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8996        let refspec = format!("{after}:refs/heads/{public}");
8997        let push = git::git_raw(
8998            &winner.worktree,
8999            &["push", &lease, &self.state.config.merge.remote, &refspec],
9000        )
9001        .await;
9002        match push {
9003            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9004            Ok(out) => {
9005                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9006                return Ok(false);
9007            }
9008            Err(e) => {
9009                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9010                return Ok(false);
9011            }
9012        }
9013        self.state.merge_repairs[i].phase = Phase::Complete;
9014        self.state.status = RunStatus::Landing;
9015        self.state.event(
9016            "repair",
9017            "new review and gate completed; fresh owner merge decision required",
9018        );
9019        self.state.save()?;
9020        Ok(true)
9021    }
9022
9023    fn merge_repair_artifacts(&self) -> PathBuf {
9024        let artifacts = agent::artifacts_dir(&self.state.dir());
9025        match self
9026            .state
9027            .merge_repairs
9028            .last()
9029            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9030        {
9031            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9032            None => artifacts,
9033        }
9034    }
9035
9036    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9037        if let Some(e2e) = self.state.head_e2e.take() {
9038            self.state
9039                .verification_history
9040                .push(crate::run::VerificationEvidence {
9041                    superseded_by: None,
9042                    archived_at: Some(Timestamp::now()),
9043                    fixes_spent: Some(self.state.gate_fixes.len()),
9044                    gate_head: None,
9045                    gate_ran: false,
9046                    gate: Vec::new(),
9047                    e2e: Some(e2e),
9048                });
9049        }
9050        self.state.reviews.clear();
9051        self.state.gate.clear();
9052        self.state.gate_ran = false;
9053        self.state.gate_head = None;
9054        self.state.gate_fixes.clear();
9055        self.state.gate_repair_e2e = None;
9056        self.state.pre_gate.clear();
9057        self.state.pre_gate_commit = None;
9058        self.state.contested_handoff = None;
9059        self.state.land_reruns = None;
9060        self.state.land_approval = None;
9061        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9062        self.state.status = RunStatus::Reviewing;
9063        self.state.save()
9064    }
9065
9066    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9067        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9068        self.state.merge_repairs[index].failure = Some(why.clone());
9069        self.state.status = RunStatus::Blocked;
9070        self.state.event("repair", why);
9071        self.state.save()
9072    }
9073
9074    // -------------------------------------------------------------- helpers
9075
9076    /// Fetch or create a seat, keeping its conversation across nodes.
9077    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9078        if let Some(existing) = self.state.seats.get(key)
9079            && existing.agent == agent
9080        {
9081            return existing.clone();
9082        }
9083        // A seat that changes agent mints its session id from the agent too,
9084        // like a handover: the old agent's uuid is already taken by the CLI.
9085        let fresh = if self.state.seats.contains_key(key) {
9086            handover_seat(key, agent, self.state.next_seat_seed())
9087        } else {
9088            SeatState::new(key, agent, self.state.seed)
9089        };
9090        self.state.seats.insert(key.to_owned(), fresh.clone());
9091        fresh
9092    }
9093
9094    /// The agent now holding seat `key`: `spec`, unless a handover moved the
9095    /// seat to another roster agent, in which case that agent. Nodes that
9096    /// continue a seat's conversation (deliberation, the votes, a reviewer's
9097    /// reconsideration) must keep talking to whoever answered it, not slip
9098    /// back to the agent that failed it.
9099    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9100        match self.state.seats.get(key) {
9101            Some(s) if s.agent != spec.id => {
9102                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9103            }
9104            _ => spec,
9105        }
9106    }
9107
9108    /// A candidate rendered for judging, with the leak policy applied.
9109    fn view(&self, c: &Candidate) -> CandidateView {
9110        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9111            .unwrap_or_default();
9112        let (patch, _) = blind::sanitize_patch(
9113            &format!("candidate {} patch", c.label),
9114            &raw,
9115            &self.state.config.blind,
9116        );
9117        CandidateView {
9118            label: c.label,
9119            branch: c.branch.clone(),
9120            summary: c.summary.clone(),
9121            stat: c.stat.clone(),
9122            patch,
9123        }
9124    }
9125
9126    /// The full candidate set as prompt text, for seats with no live session.
9127    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9128        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9129        prompt::judge(
9130            "(see above)",
9131            &views,
9132            self.roles.judges.len(),
9133            base_short,
9134            "en",
9135        )
9136    }
9137
9138    /// The final-vote prompt with everything a seat that has no session of its
9139    /// own needs: the candidates, the seat's own ranking and reasons, and the
9140    /// anonymised deliberation it took part in (only when there was one, so a
9141    /// handed-over seat never sees more than the seat it replaces did). The
9142    /// `Final vote` heading stays first.
9143    fn vote_prompt_full(
9144        &self,
9145        j: usize,
9146        labels: &[char],
9147        language: &str,
9148        candidates: &[Candidate],
9149        base_short: &str,
9150    ) -> String {
9151        let mut text = format!(
9152            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9153            prompt::final_vote(labels, language),
9154            self.state.instruction,
9155            self.candidate_block(candidates, base_short)
9156        );
9157        if let Some(own) = self
9158            .state
9159            .judgements
9160            .get(j)
9161            .filter(|r| !r.ranking.is_empty())
9162        {
9163            let reasons = own
9164                .reasons
9165                .iter()
9166                .map(|(k, v)| format!("- {k}: {v}"))
9167                .collect::<Vec<_>>()
9168                .join("\n");
9169            text.push_str(&format!(
9170                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9171                own.ranking.iter().collect::<String>(),
9172                if reasons.is_empty() {
9173                    ""
9174                } else {
9175                    ", because:\n"
9176                }
9177            ));
9178        }
9179        if !self.state.deliberation.is_empty() {
9180            text.push_str("\n# What was argued before this vote\n");
9181            for t in self.transcript(&[], j) {
9182                text.push_str(&format!(
9183                    "\n## {}{}\n\n{}\n",
9184                    t.who,
9185                    if t.is_self { " (you)" } else { "" },
9186                    t.body.trim()
9187                ));
9188            }
9189        }
9190        text
9191    }
9192
9193    /// Anonymised transcript for judge `self_idx`.
9194    ///
9195    /// The initial rankings are always the opening statements. Seeding them
9196    /// only when no turn had been taken yet meant every judge after the first
9197    /// argued against a single voice instead of against the actual split — the
9198    /// disagreement is the information, so it is always on the table.
9199    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9200        let mut turns = Vec::new();
9201        for j in &self.state.judgements {
9202            if j.ranking.is_empty() {
9203                continue;
9204            }
9205            let reasons = j
9206                .reasons
9207                .iter()
9208                .map(|(k, v)| format!("- {k}: {v}"))
9209                .collect::<Vec<_>>()
9210                .join("\n");
9211            turns.push(Turn {
9212                who: format!("Judge {} (opening ranking)", j.judge),
9213                is_self: j.judge == self_idx + 1,
9214                body: format!(
9215                    "Ranked {}{}{reasons}",
9216                    j.ranking.iter().collect::<String>(),
9217                    if reasons.is_empty() {
9218                        ""
9219                    } else {
9220                        ", because:\n"
9221                    }
9222                ),
9223            });
9224        }
9225        for t in self
9226            .state
9227            .deliberation
9228            .iter()
9229            .flat_map(|r| r.turns.iter())
9230            .chain(current)
9231        {
9232            turns.push(Turn {
9233                who: format!("Judge {}", t.judge),
9234                is_self: t.judge == self_idx + 1,
9235                body: t.body.clone(),
9236            });
9237        }
9238        turns
9239    }
9240}
9241
9242/// Does this seat still hold the context a follow-up prompt would rely on?
9243fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9244    agent::has_session(spec.kind, seat, sessions)
9245}
9246
9247/// The next entry in `roster` after `start`, never wrapping back to the
9248/// front, whose id is not in `tried` yet.
9249///
9250/// Starts one past `start` rather than at the front of `roster`: `start` is
9251/// the seat's own original position, and a seat whose candidate slot already
9252/// sits on the roster's second entry must fall through to the third next, not
9253/// restart at the first — which is very likely a different candidate's own
9254/// agent already. Never wraps back past `start`, for the same reason: an
9255/// entry earlier in the roster than the seat's own position is almost
9256/// certainly some *other* candidate slot's own agent, and once the tail of
9257/// the roster is exhausted there are no more untried agents for *this* seat
9258/// to fall through to — the caller's fallback chain ends there, exactly as
9259/// "no further untried agents remain in the list for that seat" asks for.
9260///
9261/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
9262/// the same id twice (an operator's `roles.implementers` typo, or a
9263/// `[[agents]]` list reused across roles) must not let
9264/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
9265/// over `roster` either finds an untried id or runs out, so this always
9266/// terminates regardless of duplicates.
9267fn next_untried_in_roster<'a>(
9268    roster: &'a [AgentSpec],
9269    start: usize,
9270    tried: &BTreeSet<String>,
9271) -> Option<&'a AgentSpec> {
9272    roster
9273        .get(start + 1..)?
9274        .iter()
9275        .find(|s| !tried.contains(&s.id))
9276}
9277
9278/// The successor for a seat, shared by all four seat kinds (implement, judge,
9279/// review, advise). `others` holds the ids that *currently* occupy the other
9280/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9281/// sees them), so roster entries beyond the seat count act as spares: a failed
9282/// seat goes to an agent no other seat holds whenever the roster permits.
9283///
9284/// `carried` is `Some` only for the review loop's carried failure history.
9285/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9286/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9287/// whole roster: untried and not another seat's occupant; (3) the plain walk
9288/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9289/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9290/// a better outcome than an empty seat, and it keeps the answer to "is there
9291/// a successor at all" exactly what it was before occupants were considered,
9292/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9293/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9294/// carried failure is a cheaper bet than doubling an agent on the panel, and
9295/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9296/// the same round are handled one at a time, so a seat later in the batch
9297/// sees an earlier one's new occupant but not yet its own freed agent.
9298fn pick_successor<'a>(
9299    roster: &'a [AgentSpec],
9300    start: usize,
9301    tried: &BTreeSet<String>,
9302    carried: Option<&BTreeSet<String>>,
9303    others: &BTreeSet<String>,
9304) -> Option<&'a AgentSpec> {
9305    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9306    roster
9307        .get(start + 1..)
9308        .and_then(|tail| {
9309            tail.iter()
9310                .filter(free)
9311                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9312        })
9313        .or_else(|| {
9314            carried
9315                .is_some()
9316                .then(|| roster.iter().find(free))
9317                .flatten()
9318        })
9319        .or_else(|| match carried {
9320            Some(c) => next_for_seat(roster, start, tried, c),
9321            None => next_untried_in_roster(roster, start, tried),
9322        })
9323}
9324
9325/// The next agent for a seat that carries its failure history across rounds
9326/// (the review loop). `round_tried` is this round's own bound and starts
9327/// empty every round; `carried_failed` only decides priority.
9328///
9329/// First: an id walking forward from `start`, never wrapping, that is neither
9330/// tried this round nor failed in an earlier one. Only when that is exhausted
9331/// does it rescue: the first roster id (in roster order, so this one *does*
9332/// look before `start`) not yet tried this round, which is by then a carried
9333/// failure. Each id is rescued at most once per round, so it cannot loop.
9334fn next_for_seat<'a>(
9335    roster: &'a [AgentSpec],
9336    start: usize,
9337    round_tried: &BTreeSet<String>,
9338    carried_failed: &BTreeSet<String>,
9339) -> Option<&'a AgentSpec> {
9340    roster
9341        .get(start + 1..)?
9342        .iter()
9343        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9344        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9345}
9346
9347/// Where a reviewer seat starts a round: the agent that last answered it when
9348/// it is still on the roster and not marked failed, else the spec's own agent
9349/// unless it failed, else the next roster agent that has not failed, else the
9350/// spec's own agent again (the whole roster failed). Ids no longer on the
9351/// roster are ignored. An empty roster has no handover, so the spec stands.
9352fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9353    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9354        return spec;
9355    };
9356    let ok = |id: &str| !h.failed.contains(id);
9357    if let Some(last) = h.last_ok.as_deref()
9358        && ok(last)
9359        && let Some(s) = roster.iter().find(|s| s.id == last)
9360    {
9361        return s.clone();
9362    }
9363    if ok(&spec.id) {
9364        return spec;
9365    }
9366    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9367    roster
9368        .get(from + 1..)
9369        .into_iter()
9370        .flatten()
9371        .chain(roster.iter())
9372        .find(|s| ok(&s.id))
9373        .cloned()
9374        .unwrap_or(spec)
9375}
9376
9377/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9378/// seed so a CLI that mints its session id up front (`--session-id`) never
9379/// reuses the uuid the previous agent already opened under the same seat key.
9380pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9381    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9382}
9383
9384/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9385/// so the classifier below and the code that builds the message cannot drift.
9386const TIMED_OUT: &str = "timed out";
9387
9388impl FailClass {
9389    /// `None` for an answer; otherwise how the turn failed.
9390    fn of(out: &AgentOutcome) -> Option<Self> {
9391        match out {
9392            AgentOutcome::Ok(_) => None,
9393            AgentOutcome::Quota(_) => Some(Self::Quota),
9394            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9395            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9396            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9397            // Not a failure of the agent. Callers check for it first; `None`
9398            // here means "no class to hand over on".
9399            AgentOutcome::ResourceBlocked(_) => None,
9400        }
9401    }
9402
9403    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9404    fn stem_word(&self) -> &'static str {
9405        match self {
9406            Self::Quota => "quota",
9407            _ => "handover",
9408        }
9409    }
9410}
9411
9412/// The message's first line with its variable parts removed — digit runs and
9413/// path-like tokens — so "exit code 2" and "exit code 7" read
9414/// as one kind of failure.
9415fn failure_signature(msg: &str) -> String {
9416    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9417    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9418    // the historical machine key while the process diagnosis uses readable prose.
9419    if let Some(code) = line
9420        .strip_prefix("process exited with no usable output (exit code ")
9421        .and_then(|s| s.strip_suffix(')'))
9422        .and_then(|s| s.parse::<i32>().ok())
9423    {
9424        line = format!("exited with some({code}) and no usable output");
9425    } else if matches!(
9426        line.as_str(),
9427        "process exited with no usable output (no exit code reported)"
9428            | "process exited with no usable output (exit code unavailable)"
9429    ) {
9430        line = "exited with none and no usable output".to_owned();
9431    }
9432    let mut out = Vec::new();
9433    for word in line.split_whitespace() {
9434        if word.contains('/') || word.contains('\\') {
9435            out.push("<path>".to_owned());
9436            continue;
9437        }
9438        let mut w = String::new();
9439        let mut in_digits = false;
9440        for c in word.chars() {
9441            if c.is_ascii_digit() {
9442                if !in_digits {
9443                    w.push('#');
9444                }
9445                in_digits = true;
9446            } else {
9447                in_digits = false;
9448                w.push(c);
9449            }
9450        }
9451        out.push(w);
9452    }
9453    out.join(" ").chars().take(120).collect()
9454}
9455
9456/// Whether a seat that just failed with `cur` may go to the next roster agent.
9457/// A quota or a timeout always may. Any other failure may not when the agent
9458/// before it failed the same way: an error the prompt causes would otherwise
9459/// walk the whole roster. `prev` is the class of the immediately preceding
9460/// agent's failure, so a quota or timeout in between breaks the run of
9461/// identical failures by itself.
9462fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9463    match cur {
9464        FailClass::Quota | FailClass::Timeout => true,
9465        FailClass::Other(_) => prev != Some(cur),
9466    }
9467}
9468
9469/// A short human reason for a failed outcome, for the handover record.
9470fn fail_reason(out: &AgentOutcome) -> String {
9471    match out {
9472        AgentOutcome::Ok(_) => String::new(),
9473        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9474        AgentOutcome::Dropped(o) => format!(
9475            "the CLI dropped the stream ({})",
9476            o.dropped
9477                .as_ref()
9478                .map(|d| d.why.as_str())
9479                .unwrap_or("it ended without delivering its answer")
9480        ),
9481        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9482            e.lines().next().unwrap_or("").chars().take(160).collect()
9483        }
9484    }
9485}
9486
9487/// Note one handover in the run: the structured record and, in the timeline,
9488/// the sentence a person reads. A quota keeps the wording it always had.
9489pub(crate) fn record_handover(
9490    state: &mut RunState,
9491    node: &str,
9492    seat: &str,
9493    from: &str,
9494    to: &str,
9495    class: &FailClass,
9496    reason: &str,
9497) {
9498    let message = if *class == FailClass::Quota {
9499        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9500    } else {
9501        format!("{seat}: handed over {from} -> {to} ({reason})")
9502    };
9503    state.event(node, message);
9504    state.handovers.push(Handover {
9505        at: Timestamp::now(),
9506        node: node.to_owned(),
9507        seat: seat.to_owned(),
9508        from: from.to_owned(),
9509        to: to.to_owned(),
9510        reason: reason.to_owned(),
9511    });
9512}
9513
9514/// Did this reply report running a command whose own CLI never confirmed an
9515/// exit status?
9516///
9517/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9518/// command *finished* (see that type's own doc), so this can only be `true`
9519/// for a command whose completion event carried no readable exit code — not
9520/// for one that simply is not mentioned at all. That is the one signal this
9521/// crate can read, from the same record `state.jobs` renders, about a reply
9522/// standing next to work its own CLI cannot vouch for finishing; it is
9523/// deliberately not a check on the exit code's *value* (a fixer legitimately
9524/// runs a command that fails mid-iteration before it succeeds) and not a
9525/// guess at a command still running in the background (which emits no event
9526/// at all, and so leaves no evidence here to find).
9527fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9528    commands.iter().any(|c| c.exit_code.is_none())
9529}
9530
9531/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9532/// trusted as a verified no-op — the adoption guard's own text-level half.
9533///
9534/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9535/// timed out): a marker only earns the benefit of the doubt from a turn the
9536/// CLI itself vouches for finishing properly, the same house style
9537/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9538/// *fix* report to for `commands`. A candidate that timed out, exited
9539/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9540/// is, whatever prose it wrote — this returns `None` before it ever looks at
9541/// `text`. The remaining guards (the tree really is empty, the evidence is
9542/// non-empty) are the caller's: this only reads what the reply *claimed*.
9543fn verified_noop_claim(
9544    usable: bool,
9545    commands: &[agent::CommandEvidence],
9546    text: &str,
9547) -> Option<String> {
9548    (usable && !has_unconfirmed_command(commands))
9549        .then(|| verdict::verified_noop(text))
9550        .flatten()
9551}
9552
9553fn short(commit: &str) -> String {
9554    commit.chars().take(7).collect()
9555}
9556
9557fn make_executable(path: &Path) -> Result<()> {
9558    #[cfg(unix)]
9559    {
9560        use std::os::unix::fs::PermissionsExt as _;
9561        let mut perms = std::fs::metadata(path)?.permissions();
9562        perms.set_mode(0o755);
9563        std::fs::set_permissions(path, perms)?;
9564    }
9565    #[cfg(not(unix))]
9566    {
9567        let _ = path;
9568    }
9569    Ok(())
9570}
9571
9572/// What every seat in one batch shares: where the answers are attributed, the
9573/// prompt overlay they inherit, and the build cache they are told to use.
9574///
9575/// A struct rather than four more parameters: `wave` also needs the run's
9576/// state (to record who is answering right now) and the attempt number, and
9577/// eight positional arguments is both unreadable and a clippy error.
9578struct WaveCtx<'a> {
9579    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9580    /// paid for it.
9581    run: &'a str,
9582    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9583    node: &'a str,
9584    prompts: &'a Prompts,
9585    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9586    cache: Option<&'a Path>,
9587    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9588    /// `JobRecord::round`. `None` for every other node.
9589    round: Option<usize>,
9590    /// Carry each seat's failed-agent history across waves (the review loop
9591    /// only): start-of-round priority and handover choice read
9592    /// [`RunState::seat_history`], and every answer or failure writes it.
9593    carry_seats: bool,
9594    /// The run's park / interrupt requests, so a wait for a shared resource
9595    /// can be abandoned the moment the operator asks to park.
9596    pauses: Option<(&'a Pause, &'a Pause)>,
9597}
9598
9599/// Shared scheduling policy for operations and spawned seat invocations.
9600fn repository_wait(
9601    state: &RunState,
9602    pauses: Option<(&Pause, &Pause)>,
9603) -> crate::repo_lock::WaitPolicy {
9604    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9605    let inherited = crate::repo_lock::WaitPolicy::current();
9606    crate::repo_lock::WaitPolicy::new(
9607        Duration::from_secs(state.config.graph.timeout_resource_wait),
9608        move || {
9609            inherited.cancelled()
9610                || pauses
9611                    .as_ref()
9612                    .is_some_and(|(a, b)| a.parked() || b.parked())
9613        },
9614    )
9615}
9616
9617fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9618    if error.is::<crate::repo_lock::ResourceBlocked>() {
9619        state.resource_block = Some(crate::run::ResourceBlock {
9620            cause: crate::run::ResourceBlockCause::Other,
9621            node: "repository".into(),
9622            kind: if error.to_string().contains("disk pressure") {
9623                "disk pressure"
9624            } else {
9625                "repository coordination"
9626            }
9627            .into(),
9628            detail: error.to_string(),
9629            at: Timestamp::now(),
9630        });
9631        state.event("resource", error.to_string());
9632    }
9633}
9634
9635async fn coordinate_operation(
9636    state: &mut RunState,
9637    cwd: &Path,
9638    wait: &crate::repo_lock::WaitPolicy,
9639) -> Result<Option<crate::repo_lock::SeatLease>> {
9640    let result =
9641        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9642            .await;
9643    if let Err(e) = &result {
9644        record_repository_block(state, e);
9645    }
9646    result
9647}
9648
9649/// Run one job, honouring the parallelism budget.
9650async fn run_one(
9651    job: SeatJob,
9652    sem: Arc<Semaphore>,
9653    ctx: &WaveCtx<'_>,
9654    state: &mut RunState,
9655    attempt: usize,
9656) -> (SeatState, AgentOutcome) {
9657    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9658        .await
9659        .pop()
9660        .expect("one job in, one result out");
9661    (seat, out)
9662}
9663
9664/// Run every job concurrently, capped by the semaphore, preserving order.
9665///
9666/// Every seat in the batch is recorded into [`RunState::active`] before the
9667/// wave starts and cleared as each answer lands, so the run's own record says
9668/// who is still being waited on rather than only who finished.
9669async fn wave(
9670    jobs: Vec<SeatJob>,
9671    sem: Arc<Semaphore>,
9672    ctx: &WaveCtx<'_>,
9673    state: &mut RunState,
9674    attempt: usize,
9675) -> Vec<(usize, SeatState, AgentOutcome)> {
9676    let WaveCtx {
9677        run,
9678        node,
9679        prompts,
9680        cache,
9681        round,
9682        carry_seats: _,
9683        pauses,
9684    } = *ctx;
9685    for job in &jobs {
9686        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9687    }
9688    if let Err(e) = state.save() {
9689        // A failed persist of "who is answering right now" must not abort the
9690        // wave: the seats are already being asked, and the alternative is
9691        // losing the answers to save a status line nobody may even be
9692        // watching.
9693        tracing::warn!("could not persist in-progress seats: {e:#}");
9694    }
9695    // Hold the shared build cache's lease for the whole batch, not per job:
9696    // several candidates (an implement wave) or a fixer legitimately share
9697    // one cache concurrently within this run, and that stays untouched — a
9698    // single lease taken once for the whole wave and released once it is
9699    // done is what stops a *different* borrower (another run's own wave, its
9700    // e2e/gate, a human's `magi review`) from interleaving a build into the
9701    // same directory while this one is in flight. Best-effort, not
9702    // all-or-nothing: a wave that cannot get the lease within its own
9703    // longest job's budget still runs — an hour of paid implementer calls is
9704    // not thrown away over cache contention — but every write-allowed seat
9705    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9706    // below), the same fallback a read-only seat always gets, rather than
9707    // building into a directory this run was never granted. The identity
9708    // record is still invalidated below either way, so the next tracked
9709    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9710    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9711    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9712    // never borrowed from a job's agent budget (and never charged to it either,
9713    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9714    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9715    let cache_guard = if let Some(cache_dir) = cache {
9716        if jobs_had_a_writer {
9717            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9718            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9719            match acquire_cache_lease(
9720                state,
9721                cache_dir,
9722                &owner,
9723                resource_limit,
9724                node,
9725                node,
9726                &cancel,
9727            )
9728            .await
9729            {
9730                Ok(Some(g)) => Some(g),
9731                // Parking: leave without spawning anything. The run parks at
9732                // the next boundary and the seats are asked afresh on resume.
9733                Ok(None) => {
9734                    if let Some(block) = &mut state.resource_block {
9735                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9736                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9737                        };
9738                    }
9739                    return block_wave(
9740                        jobs,
9741                        state,
9742                        node,
9743                        attempt,
9744                        "parked while waiting for the shared build cache",
9745                    );
9746                }
9747                // An unreadable lease is not contention: keep the old
9748                // fallback (no shared cache for this wave's seats).
9749                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9750                Err(e) => {
9751                    // Writing seats may not build into a cache this run does
9752                    // not hold, and an agent that cannot be given one is not
9753                    // asked at all: the run stops resumably instead.
9754                    let why = format!("{e:#}");
9755                    state.event(
9756                        "resource",
9757                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9758                    );
9759                    return block_wave(jobs, state, node, attempt, &why);
9760                }
9761            }
9762        } else {
9763            None
9764        }
9765    } else {
9766        None
9767    };
9768    // Every job keeps its whole budget: the wait above was not the agent's.
9769    // Restart the seats' clocks so "started" means "being spawned now".
9770    state.seats_spawning(node, attempt);
9771    if let Err(e) = state.save() {
9772        tracing::warn!("could not persist the start of a wave: {e:#}");
9773    }
9774    // Queueing behind this wave's own seats for a parallelism slot is normal
9775    // (each of them is bounded by its own agent timeout); only a wait beyond
9776    // all of them plus the resource limit means the slots never came back.
9777    let slot_limit = resource_limit
9778        + jobs
9779            .iter()
9780            .map(|j| j.timeout)
9781            .max()
9782            .unwrap_or_default()
9783            .saturating_mul(jobs.len() as u32);
9784    let mut set = tokio::task::JoinSet::new();
9785    // Seats announce themselves here the moment they hold a parallelism
9786    // permit, i.e. just before their CLI is spawned.
9787    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9788    let overlay = prompts.overlay(node);
9789    let repository_wait = repository_wait(state, pauses);
9790    for (i, mut job) in jobs.into_iter().enumerate() {
9791        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9792        if cache.is_some() {
9793            job.prompt.push('\n');
9794            job.prompt
9795                .push_str(&prompt::build_cache_note(node, job.allow_write));
9796        }
9797        let sem = Arc::clone(&sem);
9798        let run = run.to_owned();
9799        let node = node.to_owned();
9800        // Only implementers were told about the task's attachments, so only
9801        // their seats get the directory widened for reading.
9802        let attachments = if node == "implement" {
9803            state.attachments.clone()
9804        } else {
9805            Vec::new()
9806        };
9807        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9808        // `prompt::build_cache_note`'s doc for why setting it anyway is
9809        // exactly how a sandboxed reviewer's write refusal got reported as a
9810        // defect in the patch, not a property of its own seat. And a
9811        // write-allowed one is handed it only when the lease above was
9812        // actually acquired: a wave that could not get it (`cache_guard` is
9813        // `None`, see its own comment) must not send seats to build into a
9814        // directory this run does not hold - that is the exact concurrent,
9815        // unmanaged-write race this module exists to prevent, not something
9816        // "proceeding anyway" is allowed to reintroduce.
9817        let cache = cache
9818            .filter(|_| job.allow_write && cache_guard.is_some())
9819            .map(Path::to_path_buf);
9820        let spawned_tx = spawned_tx.clone();
9821        let repository_wait = repository_wait.clone();
9822        let coordination = state.config.coordination.clone();
9823        let fetched_base = state.fetched_base.clone();
9824        set.spawn(async move {
9825            // A wait for a parallelism slot is scheduling, not the agent: it
9826            // is bounded by its own limit and ends as a resource block, never
9827            // as a timeout of the agent that was not started.
9828            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9829                Ok(p) => p,
9830                Err(_) => {
9831                    return (
9832                        i,
9833                        job.seat,
9834                        AgentOutcome::ResourceBlocked(format!(
9835                            "no parallel slot became free within {}s",
9836                            slot_limit.as_secs()
9837                        )),
9838                        false,
9839                    );
9840                }
9841            };
9842            spawned_tx.send(job.seat.key.clone()).ok();
9843            let mut seat = job.seat;
9844            let out = repository_wait
9845                .scope(agent::invoke(
9846                    &job.spec,
9847                    &mut seat,
9848                    &Invocation {
9849                        coordination: coordination.as_ref(),
9850                        fetched_base: fetched_base.as_ref(),
9851                        cwd: &job.cwd,
9852                        prompt: &job.prompt,
9853                        timeout: job.timeout,
9854                        allow_write: job.allow_write,
9855                        sessions: job.sessions,
9856                        artifacts: &job.artifacts,
9857                        stem: &job.stem,
9858                        run: &run,
9859                        node: &node,
9860                        cache_dir: cache.as_deref(),
9861                        attachments: &attachments,
9862                        writable: &[],
9863                    },
9864                ))
9865                .await;
9866            let interrupted = out
9867                .as_ref()
9868                .err()
9869                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9870            let out = match out {
9871                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9872                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9873                // Billed work the CLI failed to hand over is not an ordinary
9874                // failure, but its text is the CLI's raw error JSON, not an
9875                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9876                // read it as one by forgetting to check. `usable()` is always
9877                // false here (dropped implies an empty response), so this has
9878                // to be checked before the catch-all `Failed` below or the
9879                // one shape this exists for is lost with the rest.
9880                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9881                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9882                Ok(o) => AgentOutcome::Failed(format!(
9883                    "process exited with no usable output ({})",
9884                    crate::notices::exit_status(o.exit_code)
9885                )),
9886                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9887                    AgentOutcome::ResourceBlocked(e.to_string())
9888                }
9889                Err(e) => AgentOutcome::Failed(e.to_string()),
9890            };
9891            (i, seat, out, interrupted)
9892        });
9893    }
9894    drop(spawned_tx);
9895    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9896    loop {
9897        let joined = tokio::select! {
9898            Some(key) = spawned_rx.recv() => {
9899                // The clock of this seat starts now, not when it queued.
9900                state.seat_spawned(&key);
9901                if let Err(e) = state.save() {
9902                    tracing::warn!("could not persist a seat's start: {e:#}");
9903                }
9904                continue;
9905            }
9906            joined = set.join_next() => joined,
9907        };
9908        let Some(joined) = joined else {
9909            break;
9910        };
9911        let (i, seat, out, interrupted) = match joined {
9912            Ok(v) => v,
9913            // No seat to clear: a panicked task never reported which one it
9914            // was. The defensive sweep below this loop is what stops that
9915            // seat's `active` entry from surviving forever.
9916            Err(e) => {
9917                tracing::error!("agent task panicked: {e}");
9918                continue;
9919            }
9920        };
9921        state.seat_finished(&seat.key);
9922        if interrupted
9923            && node == "implement"
9924            && let Some(label) = state
9925                .candidates
9926                .iter()
9927                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9928                .map(|candidate| candidate.label)
9929        {
9930            state.candidate_preservation.insert(
9931                label,
9932                "interrupted implementation awaiting persistence".into(),
9933            );
9934            state.interrupted_implementations.insert(label);
9935        }
9936        if let AgentOutcome::ResourceBlocked(why) = &out
9937            && state.resource_block.is_none()
9938        {
9939            state.resource_block = Some(crate::run::ResourceBlock {
9940                cause: crate::run::ResourceBlockCause::Other,
9941                node: node.to_owned(),
9942                kind: if why.contains("disk pressure") {
9943                    "disk pressure"
9944                } else if why.contains("repository coordination") {
9945                    "repository coordination"
9946                } else {
9947                    "parallel slot"
9948                }
9949                .to_owned(),
9950                detail: why.clone(),
9951                at: Timestamp::now(),
9952            });
9953            let activity = if interrupted {
9954                "agent execution interrupted"
9955            } else {
9956                "no agent was started"
9957            };
9958            state.event("resource", format!("{node}: {activity} - {why}"));
9959        }
9960        record_jobs(state, node, round, &seat.key, &out);
9961        if let Err(e) = state.save() {
9962            tracing::warn!("could not persist a seat's completion: {e:#}");
9963        }
9964        if collected.len() <= i {
9965            collected.resize_with(i + 1, || None);
9966        }
9967        collected[i] = Some((i, seat, out));
9968    }
9969    // Belt-and-braces for the panic branch above: every seat this exact batch
9970    // started shares this `(node, attempt)` pair, and every seat that finished
9971    // normally already cleared itself, so anything left tagged with it here
9972    // can only be a panicked task's leftover. Cleared unconditionally rather
9973    // than left to read as still answering forever.
9974    if state
9975        .active
9976        .values()
9977        .any(|a| a.node == node && a.attempt == attempt)
9978    {
9979        state
9980            .active
9981            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9982        if let Err(e) = state.save() {
9983            tracing::warn!("could not persist the end of a wave: {e:#}");
9984        }
9985    }
9986    // Whether or not the lease above was actually held, several worktrees
9987    // may just have built into the cache with nothing here able to name one
9988    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9989    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9990    // clean it might not have strictly needed; trusting a stale match would
9991    // cost it a wrong answer.
9992    if let Some(cache_dir) = cache
9993        && jobs_had_a_writer
9994    {
9995        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9996    }
9997    if let Some(guard) = cache_guard {
9998        guard.release();
9999    }
10000    collected.into_iter().flatten().collect()
10001}
10002
10003/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10004/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10005/// this is data collection, not the fix-specific completion contract in
10006/// [`Runner::continue_fix_report`], and applies regardless of which node
10007/// asked.
10008///
10009/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10010/// evidence from; `Failed` does not, and correctly contributes nothing — a
10011/// timeout or crash is not itself evidence about a command the seat may have
10012/// started.
10013fn record_jobs(
10014    state: &mut RunState,
10015    node: &str,
10016    round: Option<usize>,
10017    seat: &str,
10018    out: &AgentOutcome,
10019) {
10020    let commands: &[agent::CommandEvidence] = match out {
10021        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10022        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10023    };
10024    let checked_at = Timestamp::now();
10025    for c in commands {
10026        state.jobs.push(JobRecord {
10027            node: node.to_owned(),
10028            round,
10029            seat: seat.to_owned(),
10030            id: c.id.clone(),
10031            description: c.description.clone(),
10032            checked_at,
10033            status: match c.exit_code {
10034                Some(0) => JobStatus::Completed,
10035                Some(_) => JobStatus::Failed,
10036                None => JobStatus::Unknown,
10037            },
10038            exit_code: c.exit_code,
10039            result_summary: c.result_summary.clone(),
10040            source: c.source.clone(),
10041        });
10042    }
10043}
10044
10045/// Is a review round clean, given how many reviewer seats answered against
10046/// how many the round expected?
10047///
10048/// A seat that never answered (timeout, crash, unparsable output) is not a
10049/// seat that read the patch and found nothing — treating it as such is
10050/// exactly the bug this function exists to close. Under the default `block`
10051/// policy a missing seat can never be clean; `warn` still requires the seats
10052/// that *did* answer to have found nothing blocking and verification to be
10053/// green.
10054///
10055/// `quota_missing` narrows that `block` default for exactly one cause of
10056/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
10057/// a session limit lifts by the very next round buys nothing — the seat is
10058/// asked again with the same quota — so once every missing seat is accounted
10059/// for by a quota loss (and at least one seat *did* answer, so a decision has
10060/// something to rest on) the round is decided on the panel that could answer,
10061/// same as `warn` would. A panel that lost every seat to quota is not
10062/// decided here: `answered == 0` falls through to the existing `block`
10063/// fallback so a fully collapsed panel still waits rather than landing on no
10064/// review at all.
10065fn round_is_clean(
10066    blocking: usize,
10067    e2e_ok: bool,
10068    answered: usize,
10069    expected: usize,
10070    quota_missing: usize,
10071    policy: IncompleteReviewPolicy,
10072) -> bool {
10073    if blocking != 0 || !e2e_ok {
10074        return false;
10075    }
10076    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10077        return true;
10078    }
10079    answered > 0 && expected - answered <= quota_missing
10080}
10081
10082/// The review loop's own conclusion, derived entirely from its persisted
10083/// round records and the round budget that produced them — never from
10084/// `status`, so a reentry (or `gate`/`merge` reading it independently)
10085/// recomputes the identical answer regardless of what an earlier node in the
10086/// same walk, or a previous walk, did to `status`.
10087///
10088/// `None` while more rounds remain to try, including when review never ran
10089/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
10090/// gone clean, or the budget is spent, or the tree has stopped moving (see
10091/// [`STAGNANT_LIMIT`]), the answer is one of two things:
10092///
10093/// - An incomplete panel that raised nothing is missing input, not a
10094///   verified tree — never a hand-off candidate, whatever verification said
10095///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
10096/// - Otherwise, green e2e on the last round hands off (see
10097///   [`Runner::stop_reviewing`]); red e2e blocks.
10098///
10099/// A last round whose own verification is still `ResourceBlocked` — magi
10100/// itself never got a command to run, not evidence the patch is broken —
10101/// is neither: this returns `None` for it too, the same as "more rounds
10102/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
10103/// handling of that shape) instead of this cheap recomputation guessing a
10104/// verdict a real attempt never produced.
10105pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10106    if max_rounds == 0
10107        || reviews.last().is_some_and(|r| {
10108            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10109        })
10110    {
10111        return Some(RunStatus::Gating);
10112    }
10113    let last = reviews.last()?;
10114    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10115    if reviews.len() < max_rounds && !stagnant {
10116        return None;
10117    }
10118    if last.incomplete() && last.blocking == 0 {
10119        return Some(RunStatus::Blocked);
10120    }
10121    if last.e2e_status() == E2eStatus::ResourceBlocked
10122        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10123    {
10124        return None;
10125    }
10126    Some(if last.e2e.iter().all(CommandOutcome::ok) {
10127        RunStatus::Gating
10128    } else {
10129        RunStatus::Blocked
10130    })
10131}
10132
10133/// How long a re-ask may take, given the budget the first attempt had.
10134///
10135/// A `nudged` retry is a request to restate an answer the seat has already
10136/// worked out: it carries no new work, so it does not deserve the original
10137/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
10138/// 133 seconds while a third sat for over ten minutes on a resumed session
10139/// holding 410 KB of prior output - and because the retry had inherited the
10140/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
10141/// judging round whose other seats were long finished.
10142///
10143/// A quarter of the budget, with a floor so that a deliberately short timeout
10144/// does not collapse to nothing. A retry that re-sends the whole prompt
10145/// (because the seat kept no context) is the original job again, and keeps the
10146/// original budget.
10147pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10148    if nudged {
10149        (full / 4).max(Duration::from_secs(120)).min(full)
10150    } else {
10151        full
10152    }
10153}
10154
10155/// Run a wave and parse each reply, re-asking the seats whose reply was
10156/// unusable.
10157///
10158/// The re-ask is a nudge rather than the whole prompt again when the seat still
10159/// holds its conversation, which is the difference between a cheap retry and
10160/// paying for the entire candidate set twice.
10161///
10162/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
10163/// successor in `roster` is handed to it instead of being re-asked: the
10164/// handover is the retry. A seat with no successor left (a single-agent
10165/// roster, the roster's tail) is nudged as before, up to `retries` times. So
10166/// the asks to one seat in one node number at most
10167/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
10168/// is asked once (a dropped stream is nudged first), and only the last agent
10169/// of the chain gets the `retries` same-agent nudges. Each roster agent is
10170/// tried at most once per seat, walking forward from the seat's own position and never wrapping
10171/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
10172/// other failure stops the chain when the previous agent failed the same way
10173/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
10174/// [`has_context`] is false and the job's own full prompt and full budget are
10175/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
10176/// intermediate ones are not losses) and is returned as a failure like any
10177/// other absent seat — the caller decides whether the panel still has a
10178/// quorum. An empty `roster` disables handover: failures are nudged as they
10179/// always were, and a quota is simply lost. A reply that fails to parse or
10180/// validate is the prompt's doing and is only ever nudged, never handed over.
10181///
10182/// The returned [`SeatState`] names the agent that answered (or tried last).
10183#[allow(clippy::too_many_arguments)]
10184async fn ask_json_wave<T>(
10185    jobs: Vec<SeatJob>,
10186    sem: Arc<Semaphore>,
10187    retries: usize,
10188    roster: &[AgentSpec],
10189    ctx: &WaveCtx<'_>,
10190    losses: &mut Vec<QuotaLoss>,
10191    state: &mut RunState,
10192    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10193) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10194where
10195    T: serde::de::DeserializeOwned + Send + 'static,
10196{
10197    ask_wave_with(
10198        jobs,
10199        sem,
10200        retries,
10201        roster,
10202        ctx,
10203        losses,
10204        state,
10205        &|text: &str| {
10206            let v = verdict::extract_json::<T>(text)?;
10207            validate(&v)?;
10208            Ok(v)
10209        },
10210    )
10211    .await
10212}
10213
10214/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
10215/// a node whose answer is prose (deliberation) shares the same handover,
10216/// failure classification, quota bookkeeping and bounds instead of a copy.
10217///
10218/// A seat handed to another roster agent is sent the job's `handover` prompt
10219/// (when it has one) rather than `prompt`: the new agent has no session, so a
10220/// resume-style prompt would be incomplete. That holds for the handover ask
10221/// and for every nudge to that agent whose `has_context` is false.
10222#[allow(clippy::too_many_arguments)]
10223async fn ask_wave_with<T>(
10224    jobs: Vec<SeatJob>,
10225    sem: Arc<Semaphore>,
10226    retries: usize,
10227    roster: &[AgentSpec],
10228    ctx: &WaveCtx<'_>,
10229    losses: &mut Vec<QuotaLoss>,
10230    state: &mut RunState,
10231    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10232) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10233where
10234    T: Send + 'static,
10235{
10236    let n = jobs.len();
10237    let originals: Vec<SeatJob> = jobs;
10238    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10239    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10240    // Nudges each seat's *current* agent has taken — 0 for a first-ask
10241    // answer, N once it has gone through N nudges. Read back once this
10242    // returns, so a caller building a history record (`ReviewRecord`) can
10243    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
10244    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
10245    // field's own doc.
10246    let mut nudges: Vec<usize> = vec![0; n];
10247    // The agent now occupying each seat, the ids it has already been through,
10248    // where in the roster the walk began, the class of the last failure, and
10249    // the stem word of a handover not yet asked (full prompt, full budget).
10250    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10251    let mut tried: Vec<BTreeSet<String>> = specs
10252        .iter()
10253        .map(|s| BTreeSet::from([s.id.clone()]))
10254        .collect();
10255    let starts: Vec<usize> = specs
10256        .iter()
10257        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10258        .collect();
10259    let carry = ctx.carry_seats && !roster.is_empty();
10260    // Carried across rounds: ids that failed the seat earlier, and how the
10261    // last failure went (so a repeat of it is not handed over again).
10262    let carried: Vec<BTreeSet<String>> = originals
10263        .iter()
10264        .map(|j| {
10265            state
10266                .seat_history
10267                .get(&j.seat.key)
10268                .filter(|_| carry)
10269                .map(|h| h.failed.clone())
10270                .unwrap_or_default()
10271        })
10272        .collect();
10273    let mut prev: Vec<Option<FailClass>> = originals
10274        .iter()
10275        .map(|j| {
10276            state
10277                .seat_history
10278                .get(&j.seat.key)
10279                .filter(|_| carry)
10280                .and_then(|h| h.last_fail.clone())
10281        })
10282        .collect();
10283    let next_agent =
10284        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10285            let others: BTreeSet<String> = specs
10286                .iter()
10287                .enumerate()
10288                .filter(|(j, _)| *j != i)
10289                .map(|(_, s)| s.id.clone())
10290                .collect();
10291            pick_successor(
10292                roster,
10293                starts[i],
10294                tried,
10295                carry.then(|| &carried[i]),
10296                &others,
10297            )
10298            .cloned()
10299        };
10300    let mut fresh: Vec<Option<String>> = vec![None; n];
10301    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10302    let mut pending: Vec<usize> = (0..n).collect();
10303
10304    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10305    // asks: an agent with a successor is asked once and handed over, and only
10306    // a seat with no successor spends `retries` nudges on the same agent. This
10307    // only guarantees the loop's own termination whatever those say.
10308    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10309    for round in 0..max_rounds {
10310        if pending.is_empty() {
10311            break;
10312        }
10313        let mut batch = Vec::with_capacity(pending.len());
10314        let mut renudged: Vec<&str> = Vec::new();
10315        for &i in &pending {
10316            let src = &originals[i];
10317            // A seat now held by another agent than the job named has no
10318            // session of its own: it gets the full-context prompt whenever
10319            // it is asked in full (the handover ask, a nudge it cannot
10320            // resume).
10321            let full: &str = match &src.handover {
10322                Some(h) if specs[i].id != src.spec.id => h,
10323                _ => &src.prompt,
10324            };
10325            // The prompt and the budget are one decision: a nudge restates
10326            // finished work, a re-sent prompt redoes it.
10327            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10328                (
10329                    full.to_owned(),
10330                    src.timeout,
10331                    format!("{}-{word}-{}", src.stem, specs[i].id),
10332                )
10333            } else if nudges[i] == 0 {
10334                (full.to_owned(), src.timeout, src.stem.clone())
10335            } else {
10336                renudged.push(src.seat.key.as_str());
10337                let why = done[i]
10338                    .as_ref()
10339                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10340                    .unwrap_or_else(|| "no parsable answer".to_owned());
10341                let nudge = prompt::nudge(&why);
10342                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10343                let prompt = if nudged {
10344                    nudge
10345                } else {
10346                    format!("{full}\n\n---\n\n{nudge}")
10347                };
10348                (
10349                    prompt,
10350                    retry_budget(src.timeout, nudged),
10351                    format!("{}-retry{}", src.stem, nudges[i]),
10352                )
10353            };
10354            batch.push(SeatJob {
10355                spec: specs[i].clone(),
10356                seat: seats[i].clone(),
10357                cwd: src.cwd.clone(),
10358                prompt,
10359                timeout,
10360                allow_write: src.allow_write,
10361                sessions: src.sessions,
10362                artifacts: src.artifacts.clone(),
10363                stem,
10364                handover: None,
10365            });
10366        }
10367
10368        if !renudged.is_empty() {
10369            state.event(
10370                ctx.node,
10371                format!("retry {round}: re-asking {}", renudged.join(", ")),
10372            );
10373        }
10374        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10375        let mut still = Vec::new();
10376        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10377            seats[i] = seat;
10378            // No agent was asked: nothing to hand over, nudge, remember or
10379            // charge. The caller sees `resource_block` and stops the node.
10380            if let AgentOutcome::ResourceBlocked(why) = &out {
10381                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10382                continue;
10383            }
10384            let class = FailClass::of(&out);
10385            // A dropped stream is nudged first (the conversation is still
10386            // there to pick up); only a seat whose nudges are spent hands over.
10387            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10388                && nudges[i] < retries
10389                && !roster.is_empty();
10390            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10391                let next = should_hand_over(prev[i].as_ref(), &cur)
10392                    .then(|| next_agent(i, &tried[i], &specs))
10393                    .flatten();
10394                if carry {
10395                    let h = state
10396                        .seat_history
10397                        .entry(originals[i].seat.key.clone())
10398                        .or_default();
10399                    h.failed.insert(specs[i].id.clone());
10400                    h.last_fail = Some(cur.clone());
10401                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10402                        h.last_ok = None;
10403                    }
10404                    // Saved before the next agent is asked, so a restart in
10405                    // between does not forget who failed.
10406                    if let Err(e) = state.save() {
10407                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10408                    }
10409                }
10410                if let Some(next) = next {
10411                    record_handover(
10412                        state,
10413                        ctx.node,
10414                        &originals[i].seat.key,
10415                        &specs[i].id,
10416                        &next.id,
10417                        &cur,
10418                        &fail_reason(&out),
10419                    );
10420                    tried[i].insert(next.id.clone());
10421                    prev[i] = Some(cur.clone());
10422                    seats[i] =
10423                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10424                    specs[i] = next;
10425                    fresh[i] = Some(cur.stem_word().to_owned());
10426                    nudges[i] = 0;
10427                    done[i] = Some(Err(anyhow::anyhow!(
10428                        "handed over after: {}",
10429                        fail_reason(&out)
10430                    )));
10431                    still.push(i);
10432                    continue;
10433                }
10434            }
10435            if carry
10436                && !nudge_first
10437                && let Some(cur) = class.clone()
10438            {
10439                // The chain ended here (no successor, or a repeated failure).
10440                let h = state
10441                    .seat_history
10442                    .entry(originals[i].seat.key.clone())
10443                    .or_default();
10444                h.failed.insert(specs[i].id.clone());
10445                h.last_fail = Some(cur);
10446            }
10447            let parsed = match out {
10448                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10449                AgentOutcome::Quota(o) => {
10450                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10451                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10452                }
10453                // Not a parseable answer: the nudge loop re-asks it, which is
10454                // exactly what a dropped stream needs. Just don't hand its raw
10455                // error JSON to `extract_json`.
10456                AgentOutcome::Dropped(o) => {
10457                    let why = o
10458                        .dropped
10459                        .as_ref()
10460                        .map(|d| d.why.as_str())
10461                        .unwrap_or("the CLI ended the stream without delivering its answer");
10462                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10463                }
10464                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10465                    Err(anyhow::anyhow!(e))
10466                }
10467            };
10468            let quota = class == Some(FailClass::Quota);
10469            let failed = parsed.is_err();
10470            done[i] = Some(parsed);
10471            if carry && !failed {
10472                let h = state
10473                    .seat_history
10474                    .entry(originals[i].seat.key.clone())
10475                    .or_default();
10476                h.failed.remove(&specs[i].id);
10477                h.last_ok = Some(specs[i].id.clone());
10478                h.last_fail = None;
10479            }
10480            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10481            // fail the same way; and never re-ask a seat that already parsed.
10482            // A failed agent that still has a successor is not re-asked
10483            // either: the handover was its remedy and has just been refused
10484            // (the chain stops on a repeated failure class). A seat with no
10485            // successor left (a single-agent roster, the roster's tail, or an
10486            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10487            let agent_failure = class.is_some()
10488                && !nudge_first
10489                && !roster.is_empty()
10490                && next_agent(i, &tried[i], &specs).is_some();
10491            if failed && !quota && !agent_failure && nudges[i] < retries {
10492                nudges[i] += 1;
10493                still.push(i);
10494            }
10495        }
10496        pending = still;
10497    }
10498
10499    // One loss per seat whose chain ended on a quota: the intermediate ones
10500    // were absorbed by a handover and are not losses.
10501    for (i, q) in last_quota.into_iter().enumerate() {
10502        if let Some(reset) = q {
10503            losses.push(QuotaLoss {
10504                seat: originals[i].seat.key.clone(),
10505                node: ctx.node.to_owned(),
10506                at: Timestamp::now(),
10507                reset,
10508            });
10509        }
10510    }
10511
10512    seats
10513        .into_iter()
10514        .zip(done)
10515        .zip(nudges)
10516        .map(|((seat, res), attempts)| {
10517            (
10518                seat,
10519                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10520                attempts,
10521            )
10522        })
10523        .collect()
10524}
10525
10526/// Did the wave end without asking any agent (see [`block_wave`])?
10527fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10528    results
10529        .iter()
10530        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10531}
10532
10533/// Settle a wave that never spawned anything: every job answers
10534/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10535/// else about the seat (session, history, quota) is touched.
10536fn block_wave(
10537    jobs: Vec<SeatJob>,
10538    state: &mut RunState,
10539    node: &str,
10540    attempt: usize,
10541    why: &str,
10542) -> Vec<(usize, SeatState, AgentOutcome)> {
10543    state.resource_wait = None;
10544    state
10545        .active
10546        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10547    if let Err(e) = state.save() {
10548        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10549    }
10550    jobs.into_iter()
10551        .enumerate()
10552        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10553        .collect()
10554}
10555
10556/// Acquire the shared build cache's lease, waiting out contention within
10557/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10558/// unbounded wait is never acceptable).
10559///
10560/// A first, non-blocking check happens before ever waiting; if it finds the
10561/// lease busy, that fact is logged as a `verify` event *and* flushed with
10562/// [`RunState::save`] immediately — not only once the wait finally succeeds
10563/// or gives up — so a `magi show` run by a different process while this one
10564/// is still waiting reads a `run.json` that says so, rather than whatever it
10565/// looked like before the wait started. The same applies to the terminal
10566/// failure: logged and saved before this returns `Err`, so a caller that
10567/// could not get the lease at all still leaves a legible record of why.
10568async fn acquire_cache_lease(
10569    state: &mut RunState,
10570    cache_dir: &Path,
10571    owner: &crate::cache::Owner,
10572    budget: Duration,
10573    node: &str,
10574    context: &str,
10575    cancel: &(dyn Fn() -> bool + Sync),
10576) -> Result<Option<crate::cache::Guard>> {
10577    let home = crate::run::home();
10578    let started = Instant::now();
10579    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10580        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10581        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10582        Err(e) => {
10583            state.event(
10584                "verify",
10585                format!("{context}: could not check the shared build cache: {e:#}"),
10586            );
10587            if let Err(e2) = state.save() {
10588                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10589            }
10590            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10591        }
10592    };
10593    state.event(
10594        "verify",
10595        format!(
10596            "{context}: waiting for the shared build cache at {} ({})",
10597            cache_dir.display(),
10598            busy.describe()
10599        ),
10600    );
10601    // Seats of this node are waiting, not running: say so, so the countdown a
10602    // reader sees only ever measures an agent that has been spawned.
10603    state.resource_wait = Some(crate::run::ResourceWait {
10604        node: context.to_owned(),
10605        kind: "shared build cache".to_owned(),
10606        holder: busy.describe(),
10607        since: Timestamp::now(),
10608        limit_secs: budget.as_secs(),
10609    });
10610    if let Err(e) = state.save() {
10611        tracing::warn!("could not persist a cache wait: {e:#}");
10612    }
10613    let remaining = budget.saturating_sub(started.elapsed());
10614    let waited = crate::cache::wait_for_unless(
10615        &home,
10616        cache_dir,
10617        owner,
10618        remaining,
10619        Duration::from_secs(5),
10620        cancel,
10621    )
10622    .await;
10623    state.resource_wait = None;
10624    let cause = match &waited {
10625        Ok(Some(_)) => None,
10626        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10627        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10628            Some(crate::run::ResourceBlockCause::Timeout {
10629                waited_secs: started.elapsed().as_secs(),
10630                limit_secs: budget.as_secs(),
10631            })
10632        }
10633        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10634    };
10635    if let Some(cause) = cause {
10636        let detail = match &waited {
10637            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10638            Err(error) => format!("{error:#}"),
10639        };
10640        state.resource_block = Some(crate::run::ResourceBlock {
10641            cause,
10642            node: node.to_owned(),
10643            kind: "shared build cache".into(),
10644            detail,
10645            at: Timestamp::now(),
10646        });
10647        state.event(
10648            "resource",
10649            state.resource_block.as_ref().unwrap().description(),
10650        );
10651    }
10652    if let Err(error) = state.save() {
10653        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10654    }
10655    waited
10656}
10657
10658/// The lease state could not be read at all (unreadable lease file, a
10659/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10660/// known to be contended, so a wave degrades to running without a shared cache
10661/// instead of stopping on a hold that [`crate::triage`] could never release.
10662#[derive(Debug)]
10663struct CacheCheckError(String);
10664
10665impl std::fmt::Display for CacheCheckError {
10666    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10667        f.write_str(&self.0)
10668    }
10669}
10670
10671impl std::error::Error for CacheCheckError {}
10672
10673/// Keep a specific acquisition outcome when the caller adds command evidence.
10674fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10675    state
10676        .resource_block
10677        .get_or_insert_with(|| crate::run::ResourceBlock {
10678            cause: crate::run::ResourceBlockCause::Other,
10679            node: node.to_owned(),
10680            kind: "shared build cache".into(),
10681            detail: format!("{error:#}"),
10682            at: Timestamp::now(),
10683        });
10684    if let Err(error) = state.save() {
10685        tracing::warn!("could not persist cache resource block: {error:#}");
10686    }
10687}
10688
10689/// Run `body` — a verify command batch — while holding the shared build
10690/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10691/// never interleave with another borrower's build against the same
10692/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10693/// timeout, or a human's own `magi review`. See the `cache` module doc for
10694/// why this matters more than Cargo's own per-target locking covers — two
10695/// *different* worktrees building the same package name/version into one
10696/// cache directory is a staleness bug, not a lock contention one.
10697///
10698/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10699/// agent waves do. Repository scheduling also has its own bounded,
10700/// cancellable resource-wait policy. Contention must not consume the
10701/// verification budget: `body` receives the full `budget` after both leases
10702/// are acquired and the cache is fresh. Without a shared cache, only
10703/// repository coordination is acquired before handing over the full budget.
10704///
10705/// A lease that cannot be acquired within the resource wait is reported as a single
10706/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10707/// verification — the same shape a spawn failure already takes in
10708/// [`run_commands`], so a caller need not special-case it.
10709#[allow(clippy::too_many_arguments)]
10710async fn with_cache_lease<'s, F, Fut>(
10711    state: &'s mut RunState,
10712    cache_dir: Option<&Path>,
10713    node: &str,
10714    seat: &str,
10715    worktree: &Path,
10716    head: &str,
10717    budget: Duration,
10718    context: &str,
10719    wait: &crate::repo_lock::WaitPolicy,
10720    body: F,
10721) -> Result<(Vec<CommandOutcome>, bool)>
10722where
10723    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10724    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10725{
10726    // Match editing waves: cache first, repository second. In particular,
10727    // pre_gate must never hold the repository while waiting for a wave's cache.
10728    let started = Instant::now();
10729    let home = crate::run::home();
10730    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10731    let guard = if let Some(cache_dir) = cache_dir {
10732        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10733        match acquire_cache_lease(
10734            state,
10735            cache_dir,
10736            &owner,
10737            resource_limit,
10738            node,
10739            context,
10740            &|| wait.cancelled(),
10741        )
10742        .await
10743        .and_then(|g| {
10744            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10745        }) {
10746            Ok(g) => Some(g),
10747            Err(e) => {
10748                record_cache_error(state, node, &e);
10749                return Ok((
10750                    vec![CommandOutcome {
10751                        command: "(waiting for the shared build cache)".into(),
10752                        code: None,
10753                        output_tail: e.to_string(),
10754                        duration_ms: started.elapsed().as_millis() as u64,
10755                        resource_blocked: true,
10756                    }],
10757                    false,
10758                ));
10759            }
10760        }
10761    } else {
10762        None
10763    };
10764    let _coordination = coordinate_operation(state, worktree, wait).await?;
10765    if let Some(cache_dir) = cache_dir {
10766        let identity = crate::cache::Identity::new(worktree, head);
10767        if let Err(e) = crate::cache::ensure_fresh(
10768            &home,
10769            cache_dir,
10770            &identity,
10771            &state.config.verify.cargo_manifests(),
10772        ) {
10773            record_cache_error(state, node, &e);
10774            state.event(
10775                "verify",
10776                format!(
10777                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10778                    worktree.display(),
10779                    short(head)
10780                ),
10781            );
10782            if let Some(guard) = guard {
10783                guard.release();
10784            }
10785            return Ok((
10786                vec![CommandOutcome {
10787                    command: "(confirming the shared build cache is fresh)".into(),
10788                    code: None,
10789                    output_tail: e.to_string(),
10790                    duration_ms: started.elapsed().as_millis() as u64,
10791                    resource_blocked: true,
10792                }],
10793                false,
10794            ));
10795        }
10796    }
10797    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10798    if !timed_out_pids.is_empty() {
10799        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10800    }
10801    // Drop repository coordination before releasing the cache, reversing the
10802    // acquisition order. A cancelled/error future also drops both guards.
10803    drop(_coordination);
10804    if let Some(guard) = guard {
10805        guard.release();
10806    }
10807    Ok((outcomes, retried))
10808}
10809
10810/// Poll `pids` — commands [`run_commands`] reports as still running when its
10811/// own timeout elapsed — until every one is confirmed gone, or
10812/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10813///
10814/// Real confirmation where confirmation is possible, not a substitute for
10815/// full process-tree observation: a grandchild the timed-out process spawned
10816/// and that survives independently of it is invisible to a pid check the
10817/// same way it always was, and continuing to observe and collect *that*
10818/// stays a different piece of work with its own owner. This only narrows a
10819/// fixed blind wait into an actual check of the pids this process does know
10820/// about.
10821async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10822    wait_for_pids_with(
10823        pids,
10824        crate::proc::pid_alive,
10825        LEASE_RELEASE_POLL,
10826        LEASE_RELEASE_MAX_WAIT,
10827    )
10828    .await;
10829}
10830
10831/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10832/// interval and ceiling supplied by the caller, so the polling *logic* -
10833/// returns as soon as every pid reports dead, gives up at the ceiling
10834/// otherwise - is testable on millisecond durations without asking the real
10835/// OS about a pid at all.
10836async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10837    pids: &[u32],
10838    alive: F,
10839    poll: Duration,
10840    max_wait: Duration,
10841) {
10842    let deadline = Instant::now() + max_wait;
10843    loop {
10844        if pids.iter().all(|&pid| !alive(pid)) {
10845            return;
10846        }
10847        if Instant::now() >= deadline {
10848            return;
10849        }
10850        tokio::time::sleep(poll).await;
10851    }
10852}
10853
10854/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10855/// admission that it could not even get a verify command to run, as opposed
10856/// to evidence the command actually produced? A caller that would otherwise
10857/// read a resource-blocked outcome as a red command must check this first:
10858/// see [`Runner::gate`], which retries rather than records `Blocked` when
10859/// this is true.
10860fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10861    outcomes.iter().any(|o| o.resource_blocked)
10862}
10863
10864/// What [`Runner::gate_fix_round`] decided.
10865pub(crate) enum GateFix {
10866    /// The tree changed and `verify.e2e` is still green: run the gate again.
10867    Retry,
10868    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10869    /// last failure stands and the run ends blocked.
10870    Stop,
10871    /// `verify.e2e` could not run after the fix (magi's own contention):
10872    /// decide nothing now, a later reentry retries.
10873    Defer,
10874}
10875
10876/// Is every red command in `outcomes` an ordinary failure the code could
10877/// explain: it ran, exited non-zero, and said something?
10878///
10879/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10880/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10881/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10882/// what the command is stays the gate's business.
10883fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10884    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10885    red.peek().is_some()
10886        && red.all(|o| {
10887            !o.resource_blocked
10888                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10889                && !o.output_tail.trim().is_empty()
10890        })
10891}
10892
10893/// Describe one verify command's outcome for the event log, distinguishing a
10894/// build/link failure — the toolchain never produced a binary to run — from
10895/// an actual test failure, since only the latter is a verdict on the patch.
10896fn e2e_outcome_label(o: &CommandOutcome) -> String {
10897    if o.ok() {
10898        return "pass".to_owned();
10899    }
10900    let reason = if o.build_failed() {
10901        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10902    } else {
10903        format!("FAIL ({:?})", o.code)
10904    };
10905    format!(
10906        "{reason}\n{}",
10907        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
10908    )
10909}
10910
10911/// Run `verify.e2e`, retrying once if the first attempt could not build or
10912/// link — a build/link failure is frequently a race against a shared
10913/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10914/// `verify` event per command, tagged with `context` (normally `"round N"`)
10915/// so the two call sites that need this — the ordinary per-round leg in
10916/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10917/// it will ever call a round green — read identically in the event log.
10918async fn run_e2e_with_retry(
10919    state: &mut RunState,
10920    shell: &[String],
10921    commands: &[String],
10922    worktree: &Path,
10923    timeout: Duration,
10924    context: &str,
10925) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10926    let (mut e2e, mut timed_out_pids) = run_commands(
10927        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10928    )
10929    .await;
10930    for o in &e2e {
10931        state.event(
10932            "verify",
10933            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10934        );
10935    }
10936    // A build/link failure is not a verdict on the patch — it is frequently a
10937    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10938    // one retry before letting a red like that decide the round.
10939    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10940    if verify_retried {
10941        state.event(
10942            "verify",
10943            format!(
10944                "{context}: verify could not build/link, not a test result — retrying once \
10945                 before concluding"
10946            ),
10947        );
10948        let retried = run_commands(
10949            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10950        )
10951        .await;
10952        e2e = retried.0;
10953        // Both attempts' timeouts matter, not just the last one: the first
10954        // attempt's descendants may still be alive alongside the retry's.
10955        timed_out_pids.extend(retried.1);
10956        for o in &e2e {
10957            state.event(
10958                "verify",
10959                format!(
10960                    "{context}: retry `{}` -> {}",
10961                    o.command,
10962                    e2e_outcome_label(o)
10963                ),
10964            );
10965        }
10966    }
10967    (e2e, verify_retried, timed_out_pids)
10968}
10969
10970/// Run configured shell commands in `cwd`, in order. The second element is
10971/// the pid of every command that hit `timeout` and was still running when
10972/// this stopped waiting on it (best-effort: `None` when the platform did not
10973/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10974/// that releases a shared resource afterward needs to know.
10975///
10976/// Records `task` into [`RunState::active`] at every command boundary
10977/// (`RunState::task_command`) and clears it once the whole list has run
10978/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10979/// for minutes with no seat and no output of its own to show for it (see
10980/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10981/// apart from "not yet run" without this), and this is the only place that
10982/// knows which command is running right now and how many are left. Three
10983/// saves per command — start, not per second — matching the same "only at a
10984/// boundary" rule [`wave`] already follows for seats.
10985#[allow(clippy::too_many_arguments)]
10986async fn run_commands(
10987    state: &mut RunState,
10988    node: &str,
10989    task: &str,
10990    attempt: usize,
10991    shell: &[String],
10992    commands: &[String],
10993    cwd: &Path,
10994    timeout: Duration,
10995) -> (Vec<CommandOutcome>, Vec<u32>) {
10996    if commands.is_empty() {
10997        // Nothing to mark as running and nothing to clear — an empty list
10998        // means "not configured", and touching `active` (or the disk) over
10999        // that would be a write for every round of a repo with no
11000        // `verify.e2e` / `verify.gate` commands at all.
11001        return (Vec::new(), Vec::new());
11002    }
11003    let mut out = Vec::new();
11004    let mut timed_out_pids = Vec::new();
11005    let total = commands.len();
11006    for (idx, command) in commands.iter().enumerate() {
11007        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11008        if let Err(e) = state.save() {
11009            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11010        }
11011        let started = Instant::now();
11012        let mut cmd = tokio::process::Command::new(&shell[0]);
11013        cmd.quiet();
11014        // Verify the checkout as a standalone caller. Worker ownership and
11015        // pinned executable selection belong to this driver, not to tests or
11016        // tools it runs. Preserve home/seat context and Running's containment.
11017        for key in crate::worker::INVOCATION_ENV {
11018            cmd.env_remove(key);
11019        }
11020        cmd.args(&shell[1..])
11021            .arg(command)
11022            .current_dir(cwd)
11023            .stdin(std::process::Stdio::null())
11024            .stdout(std::process::Stdio::piped())
11025            .stderr(std::process::Stdio::piped())
11026            .kill_on_drop(true);
11027        let mut outputs = Vec::new();
11028        let protected = (|| -> Result<()> {
11029            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11030                outputs.push(guard);
11031            }
11032            if let Some(cache) = state.config.cache_dir() {
11033                outputs.push(crate::retention::OutputLease::cache(
11034                    &crate::run::home(),
11035                    &cache,
11036                    false,
11037                )?);
11038            }
11039            Ok(())
11040        })();
11041        if let Err(e) = protected {
11042            out.push(CommandOutcome {
11043                command: command.clone(),
11044                code: None,
11045                output_tail: format!("output ownership unavailable: {e:#}"),
11046                duration_ms: 0,
11047                resource_blocked: true,
11048            });
11049            continue;
11050        }
11051        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11052        let (code, body) = match spawned {
11053            Ok(mut running) => {
11054                let pid = running.child().id();
11055                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11056                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11057                let mut paths = vec![cwd.to_owned()];
11058                if let Some(cache) = state.config.cache_dir() {
11059                    paths.push(cache);
11060                }
11061                let mut pressure = None;
11062                let status = tokio::select! {
11063                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11064                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11065                        pressure = Some(reason); None
11066                    }
11067                };
11068                let (code, failure) = match status {
11069                    Some(Ok(Ok(status))) => (
11070                        status.code(),
11071                        status
11072                            .code()
11073                            .is_none()
11074                            .then(|| format!("command terminated: {status}")),
11075                    ),
11076                    Some(Ok(Err(error))) => {
11077                        (None, Some(format!("failed waiting for command: {error}")))
11078                    }
11079                    Some(Err(_)) => {
11080                        // Leased cleanup confirms process-tree death itself.
11081                        // Unleased commands still need the caller's PID check
11082                        // after kill-on-drop, before releasing cache ownership.
11083                        if !running.coordinated()
11084                            && let Some(pid) = pid
11085                        {
11086                            timed_out_pids.push(pid);
11087                        }
11088                        (
11089                            None,
11090                            Some(format!("timed out after {}s", timeout.as_secs())),
11091                        )
11092                    }
11093                    None => (None, None),
11094                };
11095                running.finish().await;
11096                if let Some(reason) = pressure {
11097                    state.resource_block = Some(crate::run::ResourceBlock {
11098                        cause: crate::run::ResourceBlockCause::Other,
11099                        node: node.to_owned(),
11100                        kind: "disk pressure".to_owned(),
11101                        detail: reason.clone(),
11102                        at: Timestamp::now(),
11103                    });
11104                    out.push(CommandOutcome {
11105                        command: command.clone(),
11106                        code: None,
11107                        output_tail: reason,
11108                        duration_ms: started.elapsed().as_millis() as u64,
11109                        resource_blocked: true,
11110                    });
11111                    break;
11112                }
11113                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11114                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11115                if let Some(failure) = failure {
11116                    body.push('\n');
11117                    body.push_str(&failure);
11118                }
11119                (code, body)
11120            }
11121            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11122        };
11123        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11124        if !body.is_empty() {
11125            // The full text is kept whatever the summary dropped. A fresh
11126            // name per command so a resume or retry never overwrites an
11127            // earlier run's output. A failed write changes no verdict.
11128            match crate::run::write_verification_artifact(state, task, &body) {
11129                Ok(name) if summary != body => {
11130                    summary = format!("[full output: artifacts/{name}]\n{summary}");
11131                }
11132                Ok(_) => {}
11133                Err(e) => {
11134                    let warning = format!("could not save the full {task} output: {e:#}");
11135                    tracing::warn!("{warning}");
11136                    state.event(node, warning);
11137                    if summary != body {
11138                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11139                    }
11140                }
11141            }
11142        }
11143        out.push(CommandOutcome {
11144            command: command.clone(),
11145            code,
11146            output_tail: summary,
11147            duration_ms: started.elapsed().as_millis() as u64,
11148            resource_blocked: false,
11149        });
11150    }
11151    state.task_finished(task);
11152    if let Err(e) = state.save() {
11153        tracing::warn!("could not persist the end of {task}: {e:#}");
11154    }
11155    (out, timed_out_pids)
11156}
11157
11158/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
11159/// section (`report::run`) and in the `merge` event this node records — for
11160/// the operator to run by hand.
11161///
11162/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
11163/// branch whose ruleset forbids merge commits (GitHub's "must not contain
11164/// merge commits", or "require linear history") rejects the push a `--no-ff`
11165/// merge would produce, which is exactly the guidance this function replaces.
11166/// `message`'s first line becomes the squash commit's subject, matching the
11167/// note `report::run` prints alongside this command — see that function for
11168/// why an explicit subject is not optional there.
11169fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11170    let repo = repo.display();
11171    match style {
11172        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11173        MergeStyle::Squash => {
11174            // The subject sits inside double quotes, and a title an agent
11175            // wrote may carry the characters that break out of them.
11176            let subject = message
11177                .lines()
11178                .next()
11179                .unwrap_or(branch)
11180                .replace(['\\', '"', '$', '`'], "");
11181            format!(
11182                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11183            )
11184        }
11185        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11186    }
11187}
11188
11189/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
11190/// under the hood, rejects a `title` over 256 characters and the whole
11191/// command fails — no PR at all, for a run whose body was otherwise fine
11192/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
11193/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
11194/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
11195/// this bound. It is a margin, not a guarantee — a title packed
11196/// with multi-unit characters could still in principle land close to the
11197/// edge, but a real task title's occasional emoji or accented letter fits
11198/// comfortably inside it.
11199const PR_TITLE_MAX: usize = 240;
11200
11201/// A pull request title from the opening line of `text`, or `None` when that
11202/// line is not English (GitHub text is) or has no letters.
11203///
11204/// A line within `max` is kept as is. A longer one is cut at the end of its
11205/// first sentence when that falls inside `max`, else at a word boundary with a
11206/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
11207/// make every merely-truncated title look non-English). The language check
11208/// runs on the kept text before any mark is added, so only what GitHub will
11209/// show is judged: an English opening followed by non-ASCII far past the cut
11210/// still passes.
11211fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11212    let line = queue::first_line(text)?;
11213    let chars: Vec<char> = line.chars().collect();
11214    let (kept, mark) = if chars.len() <= max {
11215        (line.to_owned(), "")
11216    } else if let Some(end) = sentence_end(&chars, max) {
11217        (chars[..end].iter().collect::<String>(), "")
11218    } else {
11219        let room = max.saturating_sub(3);
11220        // Cut at the last space inside the room; when the char just past the
11221        // room is a space the room already ends on a word.
11222        let cut = if chars[room].is_whitespace() {
11223            room
11224        } else {
11225            chars[..room]
11226                .iter()
11227                .rposition(|c| c.is_whitespace())
11228                .unwrap_or(room)
11229        };
11230        let head: String = chars[..cut].iter().collect();
11231        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11232        (head.to_owned(), "...")
11233    };
11234    if kept.is_empty() {
11235        return None;
11236    }
11237    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11238        return None;
11239    }
11240    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11241        return None;
11242    }
11243    Some(format!("{kept}{mark}"))
11244}
11245
11246/// The char length of the first sentence of `chars` when it ends within `max`
11247/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
11248/// abbreviations so `e.g. foo` does not end a title early.
11249fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11250    const MIN: usize = 20;
11251    for i in MIN..max.min(chars.len()) {
11252        if !matches!(chars[i], '.' | '!' | '?') {
11253            continue;
11254        }
11255        let Some(&next) = chars.get(i + 1) else {
11256            continue;
11257        };
11258        if !next.is_whitespace() {
11259            continue;
11260        }
11261        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11262        if after.is_some_and(|c| c.is_ascii_lowercase()) {
11263            continue;
11264        }
11265        let word: String = chars[..i]
11266            .iter()
11267            .rev()
11268            .take_while(|c| !c.is_whitespace())
11269            .collect::<Vec<_>>()
11270            .into_iter()
11271            .rev()
11272            .collect();
11273        let word = word.to_ascii_lowercase();
11274        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11275            continue;
11276        }
11277        let end = chars[..i]
11278            .iter()
11279            .rposition(|c| !c.is_whitespace())
11280            .map_or(i, |p| p + 1);
11281        return Some(end);
11282    }
11283    None
11284}
11285
11286/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11287/// change: a title and a body describing what was *implemented*, not the task
11288/// that asked for it. A task reads as a request; a reader of the merged
11289/// history wants the change.
11290struct PrMessage {
11291    title: String,
11292    body: String,
11293}
11294
11295impl PrMessage {
11296    /// Title, blank line, body. The first line is the squash/merge commit
11297    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11298    /// has to stay one sensible line.
11299    fn commit_message(&self) -> String {
11300        format!("{}\n\n{}", self.title, self.body)
11301    }
11302}
11303
11304/// The text after a leading `TITLE:` (any case) on `line`.
11305fn title_marker(line: &str) -> Option<&str> {
11306    let line = line.trim();
11307    let head = line.get(..6)?;
11308    head.eq_ignore_ascii_case("title:")
11309        .then(|| line[6..].trim())
11310}
11311
11312/// The implementer's own one-line title: the `TITLE:` line the implement
11313/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11314/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11315/// source, and a title that says as much is refused here too.
11316fn summary_title(summary: &str) -> Option<String> {
11317    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11318    let raw = title_marker(first)?;
11319    if raw.is_empty() {
11320        return None;
11321    }
11322    let title = queue::title_from(raw, PR_TITLE_MAX);
11323    let lower = title.to_ascii_lowercase();
11324    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11325        return None;
11326    }
11327    Some(title)
11328}
11329
11330/// How `open_review`'s instruction begins; see [`landing_title`].
11331const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11332
11333/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11334pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11335
11336/// Does this run review work that already existed, rather than implement a
11337/// task? Runs recorded before `reviewed_commits` existed carry only the
11338/// candidate marker.
11339pub(crate) fn is_review_run(state: &RunState) -> bool {
11340    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11341}
11342
11343/// The title of a review-only run: the subject of the oldest commit under
11344/// review. Later commits are usually fixups, and `instruction` is the review
11345/// prompt, which says nothing about the change. GitHub text is English, so a
11346/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11347fn review_title(state: &RunState) -> Option<String> {
11348    english_subject(
11349        state.reviewed_commits.as_ref()?.first()?,
11350        state.config.graph.github_english(),
11351    )
11352}
11353
11354/// `raw` as a pull request title, or `None` when it is blank, not English
11355/// (GitHub text is), or one of magi's own candidate commit subjects.
11356fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11357    let raw = raw.trim();
11358    if raw.is_empty() {
11359        return None;
11360    }
11361    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11362        return None;
11363    }
11364    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11365        return None;
11366    }
11367    let title = queue::title_from(raw, PR_TITLE_MAX);
11368    let lower = title.to_ascii_lowercase();
11369    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11370        return None;
11371    }
11372    Some(title)
11373}
11374
11375/// What a review-only run's branch says about itself, read at the moment the
11376/// pull request is opened.
11377#[derive(Debug, Clone, PartialEq, Eq)]
11378struct BranchFacts {
11379    /// `(subject, body)` of each commit, oldest first.
11380    commits: Vec<(String, String)>,
11381    /// Trimmed `git diff --stat`.
11382    stat: String,
11383}
11384
11385/// Longest diff stat shown: this many file lines plus the summary line.
11386const STAT_FILE_LINES: usize = 25;
11387/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11388const COMMITS_MAX_CHARS: usize = 20_000;
11389
11390/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11391/// say or finds nothing, so the caller falls back to what the run recorded.
11392async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11393    let commits = git::commit_log(repo, base, branch).await.ok()?;
11394    if commits.is_empty() {
11395        return None;
11396    }
11397    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11398    let lines: Vec<&str> = stat.lines().collect();
11399    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11400        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11401        let more = format!(" ... {omitted} more file(s)");
11402        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11403        kept.push(&more);
11404        kept.push(lines[lines.len() - 1]);
11405        kept.join("\n")
11406    } else {
11407        lines.join("\n")
11408    };
11409    Some(BranchFacts { commits, stat })
11410}
11411
11412/// Defang what would break the surrounding markdown: a closing `</details>`
11413/// and a code fence.
11414fn markdown_safe(text: &str) -> String {
11415    text.replace("</details>", "&lt;/details&gt;")
11416        .replace("\x60\x60\x60", "~~~")
11417}
11418
11419fn neutral_title(state: &RunState, winner: char) -> String {
11420    format!(
11421        "chore: land candidate {} of run {}",
11422        winner.to_ascii_uppercase(),
11423        state.id
11424    )
11425}
11426
11427/// Reuse the graph's bounded fixer without starting another graph or driver.
11428/// Copy the working state back even on errors so spent rounds remain visible.
11429pub(crate) async fn repair_landing_gate(
11430    state: &mut RunState,
11431    winner: &Candidate,
11432    outcomes: &[CommandOutcome],
11433) -> Result<GateFix> {
11434    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11435        return Ok(GateFix::Stop);
11436    }
11437    let mut runner = Runner {
11438        roles: state.config.resolve_roles()?,
11439        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11440        state: state.clone(),
11441        pause: Pause::new(),
11442        interrupt: Pause::new(),
11443    };
11444    let result = runner.gate_fix_round(winner, outcomes).await;
11445    *state = runner.state;
11446    result
11447}
11448
11449/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
11450/// A completed failure is terminal on that SHA. Resource waits remain retryable.
11451pub(crate) async fn verify_current_head(
11452    state: &mut RunState,
11453    winner: &Candidate,
11454    wait: &crate::repo_lock::WaitPolicy,
11455) -> Result<bool> {
11456    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11457    // Preserve the previous record before replacing it, including on crash recovery.
11458    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11459        state
11460            .verification_history
11461            .push(crate::run::VerificationEvidence {
11462                superseded_by: Some(head.clone()),
11463                archived_at: Some(Timestamp::now()),
11464                fixes_spent: Some(state.gate_fixes.len()),
11465                gate_head: None,
11466                gate_ran: false,
11467                gate: Vec::new(),
11468                e2e: state.head_e2e.take(),
11469            });
11470    }
11471    if let Some(e) = &state.head_e2e
11472        && e.head == head
11473    {
11474        if e.passed {
11475            return Ok(true);
11476        }
11477        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11478            state.status = RunStatus::Blocked;
11479            state.save()?;
11480            return Ok(false);
11481        }
11482    } else if state.reviews.last().is_some_and(|r| {
11483        r.verified_head.as_deref() == Some(head.as_str())
11484            && (r.e2e_status() == E2eStatus::Passed
11485                || (r.e2e_status() == E2eStatus::NotConfigured
11486                    && state.config.verify.e2e.is_empty()))
11487    }) || state
11488        .gate_repair_e2e
11489        .as_ref()
11490        .is_some_and(|e| e.head == head && e.passed)
11491    {
11492        return Ok(true);
11493    }
11494    if let Some(e) = &state.gate_repair_e2e
11495        && e.head == head
11496        && !e.passed
11497        && !e.outcomes.is_empty()
11498        && !verify_inconclusive(&e.outcomes)
11499    {
11500        state.status = RunStatus::Blocked;
11501        state.save()?;
11502        return Ok(false);
11503    }
11504    state.head_e2e = Some(crate::run::GateRepairE2e {
11505        head: head.clone(),
11506        passed: false,
11507        outcomes: Vec::new(),
11508    });
11509    state.save()?;
11510    let commands = state.config.verify.e2e.clone();
11511    let shell = state.config.shell();
11512    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11513    let cache_dir = state.config.cache_dir();
11514    let outcomes = if commands.is_empty() {
11515        Vec::new()
11516    } else {
11517        let (outcomes, _) = with_cache_lease(
11518            state,
11519            cache_dir.as_deref(),
11520            "e2e",
11521            "e2e",
11522            &winner.worktree,
11523            &head,
11524            timeout,
11525            "current head e2e",
11526            wait,
11527            |state, budget| async move {
11528                run_e2e_with_retry(
11529                    state,
11530                    &shell,
11531                    &commands,
11532                    &winner.worktree,
11533                    budget,
11534                    "current head e2e",
11535                )
11536                .await
11537            },
11538        )
11539        .await?;
11540        outcomes
11541    };
11542    let unresolved = verify_inconclusive(&outcomes);
11543    let passed = !unresolved
11544        && outcomes.iter().all(CommandOutcome::ok)
11545        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11546    state.head_e2e = Some(crate::run::GateRepairE2e {
11547        head: head.clone(),
11548        passed,
11549        outcomes,
11550    });
11551    if !passed && !unresolved {
11552        state.status = RunStatus::Blocked;
11553        state.event(
11554            "e2e",
11555            format!("verify.e2e failed on head {head}; nothing published"),
11556        );
11557    }
11558    state.save()?;
11559    Ok(passed)
11560}
11561
11562/// Recheck a repair's e2e requirement across failures, resource deferrals and
11563/// process interruptions. A successful result only applies to its exact SHA.
11564pub(crate) async fn check_gate_repair_e2e(
11565    state: &mut RunState,
11566    winner: &Candidate,
11567    wait: &crate::repo_lock::WaitPolicy,
11568) -> Result<GateFix> {
11569    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11570        return Ok(GateFix::Retry);
11571    }
11572    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11573    if state
11574        .gate_repair_e2e
11575        .as_ref()
11576        .is_some_and(|r| r.head == head && r.passed)
11577    {
11578        return Ok(GateFix::Retry);
11579    }
11580    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11581        head: head.clone(),
11582        passed: false,
11583        outcomes: Vec::new(),
11584    });
11585    state.save()?;
11586    let commands = state.config.verify.e2e.clone();
11587    let outcomes = if commands.is_empty() {
11588        Vec::new()
11589    } else {
11590        let shell = state.config.shell();
11591        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11592        let cache_dir = state.config.cache_dir();
11593        let (outcomes, _) = with_cache_lease(
11594            state,
11595            cache_dir.as_deref(),
11596            "e2e",
11597            "e2e",
11598            &winner.worktree,
11599            &head,
11600            timeout,
11601            "gate repair e2e",
11602            wait,
11603            |state, budget| async move {
11604                run_e2e_with_retry(
11605                    state,
11606                    &shell,
11607                    &commands,
11608                    &winner.worktree,
11609                    budget,
11610                    "gate repair e2e",
11611                )
11612                .await
11613            },
11614        )
11615        .await?;
11616        outcomes
11617    };
11618    let deferred = verify_inconclusive(&outcomes);
11619    let passed = outcomes.iter().all(CommandOutcome::ok)
11620        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11621    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11622        head,
11623        passed,
11624        outcomes,
11625    });
11626    state.save()?;
11627    if deferred {
11628        Ok(GateFix::Defer)
11629    } else if !passed {
11630        state.event(
11631            "gate",
11632            "verify.e2e failed after gate repair; nothing published",
11633        );
11634        Ok(GateFix::Stop)
11635    } else {
11636        Ok(GateFix::Retry)
11637    }
11638}
11639
11640/// Run the same configured gate for a landing head and a fresh graph.
11641pub(crate) async fn run_landing_gate(
11642    state: &mut RunState,
11643    winner: &Candidate,
11644    wait: &crate::repo_lock::WaitPolicy,
11645) -> Result<Vec<CommandOutcome>> {
11646    let shell = state.config.shell();
11647    let gate_commands = state.config.verify.gate.clone();
11648    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11649    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11650        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
11651        Vec::new()
11652    } else {
11653        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11654        let cache_dir = state.config.cache_dir();
11655        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11656        let (outcomes, _) = with_cache_lease(
11657            state,
11658            cache_dir.as_deref(),
11659            "gate",
11660            "gate",
11661            &winner.worktree,
11662            &head,
11663            timeout,
11664            "final gate",
11665            wait,
11666            |state, budget| {
11667                let shell = shell.clone();
11668                let gate_commands = gate_commands.clone();
11669                let winner = winner.clone();
11670                async move {
11671                    Runner::run_pre_gate(state, &winner).await;
11672                    // Bind command evidence after mechanical fixes, before commands start.
11673                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
11674                    let (outcomes, timed_out_pids) = run_commands(
11675                        state,
11676                        "gate",
11677                        "gate",
11678                        0,
11679                        &shell,
11680                        &gate_commands,
11681                        &winner.worktree,
11682                        budget,
11683                    )
11684                    .await;
11685                    (outcomes, false, timed_out_pids)
11686                }
11687            },
11688        )
11689        .await?;
11690        outcomes
11691    };
11692    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
11693    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
11694        let mut outcomes = outcomes;
11695        outcomes.push(CommandOutcome {
11696            command: "gate head stability".into(), code: None,
11697            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
11698            duration_ms: 0, resource_blocked: false,
11699        });
11700        return Ok(outcomes);
11701    }
11702    if outcomes.is_empty() {
11703        // Nothing configured to check — distinct from every other
11704        // silence in this run's event log, since an empty `gate` alone
11705        // no longer says whether the gate ran at all (see
11706        // `RunState::gate_ran`'s own doc).
11707        state.event("gate", "no gate commands configured; not tested");
11708    }
11709    for o in &outcomes {
11710        state.event(
11711            "gate",
11712            format!(
11713                "`{}` -> {}",
11714                o.command,
11715                if o.ok() {
11716                    "pass".to_owned()
11717                } else {
11718                    format!(
11719                        "FAIL ({:?})\n{}",
11720                        o.code,
11721                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11722                    )
11723                }
11724            ),
11725        );
11726    }
11727    Ok(outcomes)
11728}
11729
11730/// The pull request title to hand to `land::merge_subject`. A review-only run
11731/// opened by an earlier build titled its pull request with the review prompt;
11732/// that title is dropped (empty, so the fallback applies) rather than landed.
11733/// Any other title, including an operator's rename, passes through untouched,
11734/// and so does every title of a run that implements a task.
11735pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11736    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11737        ""
11738    } else {
11739        pr_title
11740    }
11741}
11742
11743/// What the squash subject falls back to when the pull request title is empty
11744/// or candidate-shaped: for a review-only run the derived title, never the
11745/// review prompt held in `instruction`.
11746pub fn landing_subject_source(state: &RunState) -> String {
11747    if is_review_run(state) {
11748        let winner = state.candidates.first().map_or('A', |c| c.label);
11749        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11750    }
11751    state.instruction.clone()
11752}
11753
11754/// `summary` without its `TITLE:` line, which the pull request title already
11755/// carries.
11756fn summary_without_title(summary: &str) -> String {
11757    let mut lines = summary.trim().lines().peekable();
11758    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11759        lines.next();
11760    }
11761    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11762}
11763
11764/// The pull request title and body for the winning candidate.
11765///
11766/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11767/// the task's own opening line via [`queue::title_from`] when there is none.
11768/// `state.instruction` can open with blank lines (`task_text` only rejects a
11769/// body that is blank *entirely*), which `title_from` skips.
11770///
11771/// Publishing uses an independent description generated from the final diff
11772/// and structured evidence. Internal summaries and reports remain local.
11773/// Non-publishing previews may display the original task and candidate summary.
11774/// The footer retains the run and candidate provenance.
11775#[cfg(test)]
11776fn pr_message(state: &RunState, winner: char) -> PrMessage {
11777    pr_message_with(state, winner, None)
11778}
11779
11780/// [`pr_message`] with what the branch of a review-only run says about itself.
11781/// `facts` is ignored for a run that implements a task.
11782#[cfg(test)]
11783fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11784    let raw = pr_message_raw(
11785        state,
11786        winner,
11787        facts,
11788        crate::pr_outcome::FinalHead::Unreported,
11789        ChangeNote::NotAsked,
11790    );
11791    let id = crate::scrub::Identity::current();
11792    PrMessage {
11793        title: crate::scrub::scrub(&raw.title, &id),
11794        body: crate::scrub::scrub(&raw.body, &id),
11795    }
11796}
11797
11798/// What the public description may say about the change itself.
11799#[derive(Debug, Clone, Copy)]
11800enum ChangeNote<'a> {
11801    /// A local preview or non-publishing merge mode: no public generation needed.
11802    NotAsked,
11803    /// A summary written from the final diff.
11804    Summary(&'a crate::run::ChangeSummary),
11805    /// A summary was asked for and could not be produced; the text says why.
11806    Unavailable(&'a str),
11807}
11808
11809fn pr_message_raw(
11810    state: &RunState,
11811    winner: char,
11812    facts: Option<&BranchFacts>,
11813    head: crate::pr_outcome::FinalHead<'_>,
11814    change: ChangeNote<'_>,
11815) -> PrMessage {
11816    if !matches!(change, ChangeNote::NotAsked) {
11817        let (title, mut body) = match change {
11818            ChangeNote::Summary(c) => (
11819                c.title.clone(),
11820                format!("## Summary\n\n{}\n\n", c.body.trim()),
11821            ),
11822            ChangeNote::Unavailable(why) => (
11823                neutral_title(state, winner),
11824                format!(
11825                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11826                ),
11827            ),
11828            ChangeNote::NotAsked => unreachable!(),
11829        };
11830        body.push_str(&public_pr_evidence(state, head));
11831        body.push_str(&format!(
11832            "\n---\nmagi:run/{} magi:candidate-{}\n",
11833            state.id,
11834            winner.to_ascii_lowercase()
11835        ));
11836        return PrMessage { title, body };
11837    }
11838    pr_preview(state, winner, facts, head)
11839}
11840
11841/// Append public facts from the run records after every prose generation,
11842/// including posting-gate rewrites. The agent cannot drop this provenance.
11843fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11844    let mut body = crate::pr_outcome::review_rounds(state);
11845    body.push_str(&crate::pr_outcome::final_verification(state, head));
11846    let open = state.last_round_findings();
11847    if !open.is_empty() {
11848        body.push_str("## Open review findings\n\n");
11849        for f in open {
11850            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11851        }
11852        body.push('\n');
11853    }
11854    if let Some(c) = state.change_summary.as_ref()
11855        && head == crate::pr_outcome::FinalHead::At(&c.head)
11856    {
11857        body.push_str(&format!(
11858            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11859            c.base, c.head
11860        ));
11861    }
11862    body
11863}
11864
11865/// Local-only preview of the operator's task and candidate summary.
11866fn pr_preview(
11867    state: &RunState,
11868    winner: char,
11869    facts: Option<&BranchFacts>,
11870    head: crate::pr_outcome::FinalHead<'_>,
11871) -> PrMessage {
11872    let summary = state
11873        .candidates
11874        .iter()
11875        .find(|c| c.label == winner)
11876        .map(|c| c.summary.as_str())
11877        .unwrap_or_default();
11878    // The fallback is the operator's own words and may not be English; GitHub
11879    // text always is, so a non-English task gets a neutral title instead.
11880    let review = is_review_run(state);
11881    let title = if review {
11882        facts
11883            .and_then(|f| {
11884                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11885            })
11886            .or_else(|| review_title(state))
11887            .or_else(|| {
11888                state
11889                    .candidates
11890                    .iter()
11891                    .find(|c| c.label == winner)
11892                    .filter(|c| !c.branch.starts_with("magi/"))
11893                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11894            })
11895            .unwrap_or_else(|| neutral_title(state, winner))
11896    } else {
11897        summary_title(summary).unwrap_or_else(|| {
11898            english_title(
11899                &state.instruction,
11900                PR_TITLE_MAX,
11901                state.config.graph.github_english(),
11902            )
11903            .unwrap_or_else(|| neutral_title(state, winner))
11904        })
11905    };
11906
11907    let mut body = String::new();
11908    let what = summary_without_title(summary);
11909    if !what.is_empty() {
11910        body.push_str("## Summary\n\n");
11911        body.push_str(&what);
11912        body.push_str("\n\n");
11913    }
11914
11915    // Review history uses structured facts across all rounds, never reports.
11916    body.push_str(&crate::pr_outcome::review_rounds(state));
11917    body.push_str(&crate::pr_outcome::final_verification(state, head));
11918    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11919
11920    let open = state.open_findings();
11921    if !open.is_empty() {
11922        body.push_str("## Open review findings\n\n");
11923        for f in &open {
11924            body.push_str(&format!(
11925                "- `{}` [{:?}] {}\n",
11926                f.id, f.severity, "Context retained in local review records."
11927            ));
11928        }
11929        body.push('\n');
11930    }
11931
11932    if let Some(fix) = fix
11933        && !fix.rejected.is_empty()
11934    {
11935        body.push_str("## Declined by the fixer\n\n");
11936        for r in &fix.rejected {
11937            body.push_str(&format!(
11938                "- `{}`: {}\n",
11939                r.id, "Reason retained in local review records."
11940            ));
11941        }
11942        body.push('\n');
11943    }
11944
11945    if review {
11946        // The review prompt is not the task; list what the branch carries.
11947        body.push_str("## Commits under review\n\n");
11948        if let Some(facts) = facts {
11949            let mut left = COMMITS_MAX_CHARS;
11950            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11951                let mut entry = format!("- {}\n", markdown_safe(subject));
11952                for l in markdown_safe(text).lines() {
11953                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11954                }
11955                if left == 0 {
11956                    body.push_str(&format!(
11957                        "- ... {} more commit(s)\n",
11958                        facts.commits.len() - i
11959                    ));
11960                    break;
11961                }
11962                if entry.len() > left {
11963                    // Even the first commit is cut: one huge body must not
11964                    // push the whole description past GitHub's limit.
11965                    let mut end = left;
11966                    while !entry.is_char_boundary(end) {
11967                        end -= 1;
11968                    }
11969                    entry.truncate(end);
11970                    entry.push_str("\n  ... (truncated)\n");
11971                    left = 0;
11972                } else {
11973                    left -= entry.len();
11974                }
11975                body.push_str(&entry);
11976            }
11977            if !facts.stat.trim().is_empty() {
11978                body.push_str(&format!(
11979                    "\n## Diff stat\n\n```\n{}\n```\n",
11980                    markdown_safe(facts.stat.trim())
11981                ));
11982            }
11983        } else {
11984            match &state.reviewed_commits {
11985                Some(subjects) => {
11986                    for s in subjects {
11987                        body.push_str(&format!("- {}\n", s.trim()));
11988                    }
11989                }
11990                None => {
11991                    // An older run kept only the prompt, with the commit list
11992                    // after its first paragraph.
11993                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11994                    body.push_str(rest.trim());
11995                    body.push('\n');
11996                }
11997            }
11998        }
11999    } else {
12000        let task = state.instruction.trim();
12001        let task = if task.is_empty() {
12002            "(empty task)"
12003        } else {
12004            task
12005        };
12006        body.push_str(&format!(
12007            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12008            task.replace("</details>", "&lt;/details&gt;")
12009        ));
12010    }
12011
12012    body.push_str(&format!(
12013        "\n---\nmagi:run/{} magi:candidate-{}\n",
12014        state.id,
12015        winner.to_ascii_lowercase()
12016    ));
12017
12018    PrMessage { title, body }
12019}
12020
12021/// Source imports have already been prepared from structured inputs. Legacy
12022/// inferred references must never instruct a seat to reconstruct those imports.
12023fn seeded_instruction(state: &RunState) -> String {
12024    let mut inputs = Vec::new();
12025    if let Some(start) = &state.sources.start_from {
12026        inputs.push(format!(
12027            "Start from `{}` at `{}` (including ancestry).",
12028            start.token, start.sha
12029        ));
12030    }
12031    for pick in &state.sources.pick_commits {
12032        inputs.push(format!(
12033            "Pick `{}` at `{}` (single commit).",
12034            pick.token, pick.sha
12035        ));
12036    }
12037    if inputs.is_empty() {
12038        state.instruction.clone()
12039    } else {
12040        format!(
12041            "{}\n\n# Explicit source inputs already prepared\n\n{}",
12042            state.instruction,
12043            inputs.join("\n")
12044        )
12045    }
12046}
12047
12048/// Does the winner have no commits ahead of the base it would land on?
12049/// Any failure to find out reads as "not empty": the merge then behaves as it
12050/// always did rather than refusing on a guess.
12051async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12052    let base = &state.base_branch;
12053    let mut against = base.clone();
12054    if mode == MergeMode::Pr {
12055        // A pull request lands on the remote's base, never the local branch
12056        // of the same name: if that cannot be read, "not empty" is the safe
12057        // answer.
12058        let remote = &state.config.merge.remote;
12059        let tracking = format!("{remote}/{base}");
12060        let fetched = git::fetch(repo, remote, base).await;
12061        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12062            against = tracking;
12063        } else {
12064            return false;
12065        }
12066    }
12067    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12068}
12069
12070/// Why nothing was opened for an empty winner, with what the task's own
12071/// references resolved to.
12072fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12073    let mut detail = format!(
12074        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12075         nothing to open a pull request for"
12076    );
12077    detail.push_str(
12078        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12079    );
12080    detail
12081}
12082
12083/// What the `Pr` merge does once it knows whether the branch already has an
12084/// open pull request.
12085#[derive(Debug, PartialEq, Eq)]
12086enum PrPlan {
12087    Create,
12088    Adopt { url: String, title: String },
12089    Stop(String),
12090}
12091
12092/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
12093/// a failed lookup -> stop with the real reason. Never guesses.
12094fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12095    match found {
12096        Ok(land::OpenPr::None) => PrPlan::Create,
12097        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12098        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12099            "several open pull requests exist for this branch, not picking one: {}",
12100            urls.join(" ")
12101        )),
12102        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12103    }
12104}
12105
12106/// `gh pr create`, returning the PR url.
12107pub(crate) async fn gh_pr_create(
12108    cwd: &Path,
12109    base: &str,
12110    head: &str,
12111    title: &str,
12112    body: &str,
12113) -> Result<String> {
12114    let out = tokio::process::Command::new("gh")
12115        .args([
12116            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12117        ])
12118        .current_dir(cwd)
12119        .quiet()
12120        .stdin(std::process::Stdio::null())
12121        .output()
12122        .await
12123        .context("spawn gh")?;
12124    if out.status.success() {
12125        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12126    } else {
12127        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12128    }
12129}
12130
12131/// Never force deletion of uncommitted work, including unknown legacy trees.
12132async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12133    match std::fs::symlink_metadata(path) {
12134        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12135            // A previous cleanup may have removed the tree before a branch
12136            // deletion or state save failed. Unregister it if still present.
12137            git::worktree_remove_clean(repo, path).await?;
12138            return Ok(());
12139        }
12140        Err(e) => return Err(e.into()),
12141        Ok(_) => {}
12142    }
12143    if !git::is_clean(path).await? {
12144        bail!(
12145            "worktree has uncommitted work; refusing removal: {}",
12146            path.display()
12147        );
12148    }
12149    if !git::worktree_remove_clean(repo, path).await? {
12150        bail!("worktree removal refused: {}", path.display());
12151    }
12152    Ok(())
12153}
12154
12155/// Tear a run's worktrees and branches down.
12156///
12157/// `home` is where the updated `run.json` is saved (via
12158/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
12159/// a housekeeping pass already has its own honest `home` handed to it, and
12160/// falling through to the global here would write back through whichever
12161/// directory some other process or test pinned into that `OnceLock` first,
12162/// not the one the caller actually resolved its `runs` and `state` from.
12163pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12164    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12165        bail!("refusing fold: {reason}");
12166    }
12167    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12168        bail!("candidate preservation incomplete; refusing cleanup");
12169    }
12170    // Serialize ordinary folding with the same starts/resumes as target-only
12171    // eviction. Otherwise a second janitor could remove the whole worktree
12172    // around a live output borrower even though target eviction was fenced.
12173    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12174    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12175    let _driver = crate::driver::maintenance(home, &state.id)?;
12176    let mut output_guards = Vec::new();
12177    for entry in std::fs::read_dir(state.worktree_root())
12178        .into_iter()
12179        .flatten()
12180        .flatten()
12181    {
12182        if entry.path().join(".git").is_file() {
12183            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12184        }
12185    }
12186    let repo = state.repo.clone();
12187    // CLI, Web and automatic cleanup all enter here, independently of
12188    // Runner::fold_losers. Protect deletion and shared config release too.
12189    let wait = repository_wait(state, None);
12190    let _coordination = match coordinate_operation(state, &repo, &wait).await {
12191        Ok(lease) => lease,
12192        Err(e) => {
12193            state.save_under(home)?;
12194            return Err(e);
12195        }
12196    };
12197    // Confirmed merges dispose of their owned sources after durable evidence
12198    // capture. Preflight every source before removing anything.
12199    let _source_outputs = if state.status == RunStatus::Merged
12200        && state.merge.as_ref().is_some_and(|merge| merge.ok)
12201    {
12202        verify_merged_fold_sources(state, drop_winner, home).await?
12203    } else {
12204        Vec::new()
12205    };
12206    let root = state.worktree_root();
12207    let winner = state.tally.as_ref().map(|t| t.winner);
12208    let mut removed = Vec::new();
12209
12210    for i in 0..state.candidates.len() {
12211        let c = state.candidates[i].clone();
12212        let is_winner = Some(c.label) == winner;
12213        if is_winner && !drop_winner {
12214            continue;
12215        }
12216        let existed = c.worktree.exists();
12217        // A vanished directory can still have a Git registration that keeps
12218        // its branch checked out. Remove that registration before the branch.
12219        remove_preserved_worktree(&repo, &c.worktree).await?;
12220        if existed {
12221            removed.push(c.worktree.to_string_lossy().into_owned());
12222        }
12223        // A branch handed to a later run (and its pull request) is not this
12224        // run's to delete.
12225        let handed_over = state.released_branches.contains(&c.branch);
12226        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12227            if !git::branch_delete(&repo, &c.branch).await? {
12228                bail!("candidate branch removal refused");
12229            }
12230            removed.push(c.branch.clone());
12231        }
12232        state.candidates[i].folded = true;
12233    }
12234
12235    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12236        let path = name.path();
12237        let keep = !drop_winner
12238            && winner.is_some_and(|w| {
12239                path.file_name()
12240                    .is_some_and(|n| n == format!("cand-{w}").as_str())
12241            });
12242        if keep {
12243            continue;
12244        }
12245        remove_preserved_worktree(&repo, &path).await?;
12246        removed.push(path.to_string_lossy().into_owned());
12247    }
12248
12249    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
12250    // judge worktrees, so once the loop above has cleared all of them out,
12251    // the parent is a bare directory nobody else was ever going to remove -
12252    // git only ever managed what was inside it. Left alone, one of these
12253    // accumulates per fully-folded run; the operator's own machine had 74.
12254    // `remove_if_empty` re-checks rather than assuming: a run whose winner
12255    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
12256    // so does anything a run never claimed that happens to share the bay.
12257    remove_if_empty(&root);
12258
12259    if state.enabled_worktree_config && drop_winner {
12260        // A release, not a raw disable: some sibling run in this repository
12261        // may still hold its own reference (see `git::acquire_worktree_config`),
12262        // and only the last release actually turns the setting back off.
12263        git::release_worktree_config(&repo).await.ok();
12264        state.enabled_worktree_config = false;
12265    }
12266    state.save_under(home)?;
12267    Ok(removed)
12268}
12269
12270async fn verify_merged_fold_sources(
12271    state: &RunState,
12272    drop_winner: bool,
12273    home: &Path,
12274) -> Result<Vec<std::fs::File>> {
12275    let winner = state.tally.as_ref().map(|t| t.winner);
12276    let mut heads = BTreeSet::new();
12277    for candidate in &state.candidates {
12278        if Some(candidate.label) == winner && !drop_winner {
12279            continue;
12280        }
12281        anyhow::ensure!(
12282            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12283            "candidate recovery source is outside the inventoried run root: {}",
12284            candidate.worktree.display()
12285        );
12286        match std::fs::symlink_metadata(&candidate.worktree) {
12287            Ok(_) => {}
12288            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12289            Err(error) => return Err(error.into()),
12290        }
12291        if !state.released_branches.contains(&candidate.branch)
12292            && git::branch_exists(&state.repo, &candidate.branch).await?
12293        {
12294            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12295        }
12296    }
12297    let mut cargo_guards = Vec::new();
12298    let entries = match std::fs::read_dir(state.worktree_root()) {
12299        Ok(entries) => Some(entries),
12300        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12301        Err(error) => return Err(error.into()),
12302    };
12303    for entry in entries.into_iter().flatten() {
12304        let path = entry?.path();
12305        let metadata = std::fs::symlink_metadata(&path)?;
12306        anyhow::ensure!(
12307            metadata.is_dir() && !metadata.file_type().is_symlink(),
12308            "non-directory or symlink recovery source retained: {}",
12309            path.display()
12310        );
12311        if !drop_winner
12312            && winner.is_some_and(|w| {
12313                path.file_name()
12314                    .is_some_and(|name| name == format!("cand-{w}").as_str())
12315            })
12316        {
12317            continue;
12318        }
12319        anyhow::ensure!(
12320            path.join(".git").is_file(),
12321            "unregistered recovery source retained: {}",
12322            path.display()
12323        );
12324        anyhow::ensure!(
12325            git::git(
12326                &path,
12327                &[
12328                    "status",
12329                    "--porcelain",
12330                    "--untracked-files=all",
12331                    "--ignored",
12332                    "--",
12333                    ".",
12334                    ":(exclude)target"
12335                ]
12336            )
12337            .await?
12338            .is_empty(),
12339            "source changes or ignored recovery files retained: {}",
12340            path.display()
12341        );
12342        match std::fs::symlink_metadata(path.join("target")) {
12343            Ok(_) => {
12344                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12345            }
12346            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12347            Err(error) => return Err(error.into()),
12348        }
12349        for checkpoint in [
12350            "rebase-merge",
12351            "rebase-apply",
12352            "sequencer",
12353            "CHERRY_PICK_HEAD",
12354            "MERGE_HEAD",
12355            "REBASE_HEAD",
12356            "REVERT_HEAD",
12357            "BISECT_LOG",
12358            "AUTO_MERGE",
12359            "MERGE_MSG",
12360        ] {
12361            let location = git::git(
12362                &path,
12363                &[
12364                    "rev-parse",
12365                    "--path-format=absolute",
12366                    "--git-path",
12367                    checkpoint,
12368                ],
12369            )
12370            .await?;
12371            anyhow::ensure!(
12372                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12373                if e.kind() == std::io::ErrorKind::NotFound),
12374                "Git recovery checkpoint retained: {checkpoint}"
12375            );
12376        }
12377        heads.insert(git::rev_parse(&path, "HEAD").await?);
12378    }
12379    crate::fold_evidence::preserve(state, home, &heads).await?;
12380
12381    Ok(cargo_guards)
12382}
12383
12384/// Remove `dir` if it exists and has nothing in it.
12385///
12386/// Best-effort and silent by design: a directory that is not empty (a run
12387/// whose winner is still parked there, a stray file some other process left)
12388/// is exactly the case this must refuse, and a directory that is already gone
12389/// is not a failure worth reporting either. `std::fs::remove_dir` itself
12390/// already refuses a non-empty directory, so the emptiness check below is
12391/// belt, not suspenders - it is what keeps this from ever attempting the
12392/// removal in the case that matters, rather than trusting `remove_dir`'s
12393/// error path to have no side effects if it ever changed.
12394fn remove_if_empty(dir: &Path) {
12395    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12396        std::fs::remove_dir(dir).ok();
12397    }
12398}
12399
12400/// Severity of the worst open finding in the last review round, for reporting.
12401pub fn worst_open(state: &RunState) -> Option<Severity> {
12402    state
12403        .reviews
12404        .last()?
12405        .reviews
12406        .iter()
12407        .flat_map(|r| r.findings.iter())
12408        .map(|f| f.severity)
12409        .max()
12410}
12411
12412/// `[worktree] setup` hides its products per worktree, which needs
12413/// `extensions.worktreeConfig`. Taken once per run and released with the run's
12414/// worktrees, like the hook's; every place that runs setup calls this first.
12415async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12416    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12417        git::acquire_worktree_config(repo).await?;
12418        state.enabled_worktree_config = true;
12419    }
12420    Ok(())
12421}
12422
12423#[cfg(test)]
12424mod tests {
12425    #[tokio::test]
12426    async fn launcher_base_fetch_requires_remote_success() {
12427        let tmp = tempfile::tempdir().unwrap();
12428        let repo = tmp.path().join("repo");
12429        let remote = tmp.path().join("remote");
12430        std::fs::create_dir(&repo).unwrap();
12431        std::fs::create_dir(&remote).unwrap();
12432        git::git(&remote, &["init", "--bare", "-b", "main"])
12433            .await
12434            .unwrap();
12435        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12436        git::git(
12437            &repo,
12438            &[
12439                "-c",
12440                "user.name=Fixture",
12441                "-c",
12442                "user.email=fixture@example.invalid",
12443                "commit",
12444                "--allow-empty",
12445                "-m",
12446                "fixture",
12447            ],
12448        )
12449        .await
12450        .unwrap();
12451        git::git(
12452            &repo,
12453            &["remote", "add", "origin", remote.to_str().unwrap()],
12454        )
12455        .await
12456        .unwrap();
12457        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12458        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12459        assert_eq!(
12460            resolve_base(&repo, "main", "origin", None)
12461                .await
12462                .unwrap()
12463                .commit,
12464            head
12465        );
12466        #[cfg(unix)]
12467        {
12468            let policy: crate::config::Coordination = toml::from_str(
12469                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12470            ).unwrap();
12471            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12472            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12473            let held = crate::repo_lock::SeatLease::operation_with(
12474                &repo,
12475                Some(&policy),
12476                &crate::repo_lock::WaitPolicy::default(),
12477            )
12478            .await
12479            .unwrap();
12480            assert!(held.is_some());
12481            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12482            tokio::pin!(fetch);
12483            assert!(
12484                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12485                    .await
12486                    .is_err()
12487            );
12488            assert!(
12489                !repo.join(".git/FETCH_HEAD").exists(),
12490                "no fetch before admission"
12491            );
12492            drop(held);
12493            let fetched = fetch.await.unwrap();
12494            assert_eq!(fetched.commit, head);
12495            assert_eq!(fetched.remote, "origin");
12496            assert_eq!(fetched.branch, "main");
12497        }
12498        // A surviving tracking ref is not successful fetch evidence.
12499        git::git(
12500            &repo,
12501            &[
12502                "remote",
12503                "set-url",
12504                "origin",
12505                tmp.path().join("missing").to_str().unwrap(),
12506            ],
12507        )
12508        .await
12509        .unwrap();
12510        let error = resolve_base(&repo, "main", "origin", None)
12511            .await
12512            .unwrap_err()
12513            .to_string();
12514        assert!(error.contains("cannot read origin/main"));
12515        assert!(error.contains("refusing to branch off"));
12516    }
12517
12518    #[test]
12519    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12520        let leaked = vec!["chore(deps): update a crate".to_owned()];
12521        let own = "fix(daemon): apply a chosen action";
12522        assert!(should_retitle("", own, &leaked));
12523        assert!(should_retitle(
12524            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12525            own,
12526            &leaked
12527        ));
12528        assert!(should_retitle(
12529            "chore: land candidate A of run 1",
12530            own,
12531            &leaked
12532        ));
12533        assert!(should_retitle(
12534            "magi: candidate A (uncommitted work)",
12535            own,
12536            &leaked
12537        ));
12538        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12539        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12540        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12541    }
12542
12543    #[test]
12544    fn pr_merge_plan_creates_adopts_or_stops() {
12545        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12546        assert_eq!(
12547            pr_merge_plan(Ok(land::OpenPr::One {
12548                url: "u".into(),
12549                title: "t".into()
12550            })),
12551            PrPlan::Adopt {
12552                url: "u".into(),
12553                title: "t".into()
12554            }
12555        );
12556        let PrPlan::Stop(many) =
12557            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12558        else {
12559            panic!("many must stop");
12560        };
12561        assert!(many.contains('a') && many.contains('b'));
12562        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12563            panic!("a failed lookup must stop");
12564        };
12565        assert!(err.contains("bad token"));
12566    }
12567
12568    use super::*;
12569    use crate::run::GateStatus;
12570    use std::collections::BTreeMap;
12571    use std::time::Duration;
12572
12573    fn conductor() -> AgentSpec {
12574        AgentSpec {
12575            id: "conductor".to_owned(),
12576            kind: crate::config::AgentKind::Command,
12577            model: None,
12578            command: vec!["true".to_owned()],
12579            extra_args: Vec::new(),
12580            env: BTreeMap::new(),
12581            prompt_delivery: None,
12582            max_turn_secs: None,
12583        }
12584    }
12585
12586    fn spec(id: &str) -> AgentSpec {
12587        AgentSpec {
12588            id: id.to_owned(),
12589            kind: crate::config::AgentKind::Command,
12590            model: None,
12591            command: vec!["true".to_owned()],
12592            extra_args: Vec::new(),
12593            env: BTreeMap::new(),
12594            prompt_delivery: None,
12595            max_turn_secs: None,
12596        }
12597    }
12598
12599    fn ids(xs: &[&str]) -> BTreeSet<String> {
12600        xs.iter().map(|s| (*s).to_owned()).collect()
12601    }
12602
12603    #[test]
12604    fn pick_successor_skips_an_agent_another_seat_holds() {
12605        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12606        let roster = [spec("a"), spec("b"), spec("c")];
12607        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12608        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12609    }
12610
12611    #[test]
12612    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12613        // The other seat started on c but was handed to d; c is free again.
12614        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12615        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12616        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12617        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12618        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12619    }
12620
12621    #[test]
12622    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12623        let roster = [spec("a"), spec("b")];
12624        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12625        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12626        let carried = ids(&["b"]);
12627        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12628        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12629    }
12630
12631    #[test]
12632    fn pick_successor_returns_none_without_any_untried_successor() {
12633        let roster = [spec("a"), spec("b")];
12634        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12635        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12636        let carried = ids(&["a"]);
12637        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12638    }
12639
12640    #[test]
12641    fn pick_successor_rescue_avoids_another_seats_occupant() {
12642        // b is a carried failure and free; a is held by the other seat.
12643        let roster = [spec("a"), spec("b"), spec("c")];
12644        let carried = ids(&["b", "c"]);
12645        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12646        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12647    }
12648
12649    #[test]
12650    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12651        let roster = [spec("a"), spec("b"), spec("c")];
12652        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12653        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12654    }
12655
12656    #[test]
12657    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12658        let roster = [spec("a"), spec("b"), spec("c")];
12659        let failed = ids(&["a", "b", "c"]);
12660        // Rescue looks at the whole roster, once per id, then runs out.
12661        let mut tried = ids(&["b"]);
12662        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12663        assert_eq!(first.id, "a");
12664        tried.insert(first.id.clone());
12665        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12666        assert_eq!(second.id, "c");
12667        tried.insert(second.id.clone());
12668        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12669    }
12670
12671    #[test]
12672    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12673        let roster = [spec("a"), spec("b")];
12674        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12675        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12676    }
12677
12678    #[test]
12679    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12680        let roster = [spec("a"), spec("b"), spec("c")];
12681        let h = SeatHistory {
12682            failed: ids(&["a"]),
12683            last_ok: Some("b".to_owned()),
12684            last_fail: None,
12685        };
12686        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12687        // A last answerer that left the roster, or later failed, is ignored.
12688        let gone = SeatHistory {
12689            last_ok: Some("zzz".to_owned()),
12690            ..h.clone()
12691        };
12692        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12693        let failed = SeatHistory {
12694            failed: ids(&["a", "b"]),
12695            last_ok: Some("b".to_owned()),
12696            last_fail: None,
12697        };
12698        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12699    }
12700
12701    #[test]
12702    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12703        let first = SeatState::new("review-1", "alpha", 7);
12704        let next = handover_seat("review-1", "gamma", 7);
12705        assert_ne!(first.claude_session, next.claude_session);
12706    }
12707
12708    #[test]
12709    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12710        let mut state = state_with_summary("x", "y");
12711        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12712        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12713        assert_ne!(a.claude_session, b.claude_session);
12714    }
12715
12716    #[test]
12717    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12718        let roster = [spec("a"), spec("b")];
12719        let h = SeatHistory {
12720            failed: ids(&["a", "b"]),
12721            ..SeatHistory::default()
12722        };
12723        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12724        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12725        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12726    }
12727
12728    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12729    // fallback loop depends on to terminate: it must walk forward from the
12730    // seat's own position, never restart at the front of the roster, and it
12731    // must never hand back an id already tried, however many times that id
12732    // happens to appear.
12733
12734    #[test]
12735    fn failure_signature_ignores_numbers_and_paths() {
12736        assert_eq!(
12737            failure_signature("exited with Some(2) and no usable output"),
12738            failure_signature("exited with Some(137) and no usable output")
12739        );
12740        assert_eq!(
12741            failure_signature("process exited with no usable output (exit code 2)"),
12742            failure_signature("process exited with no usable output (exit code 137)")
12743        );
12744        for (old, new) in [
12745            (
12746                "exited with Some(2) and no usable output",
12747                "process exited with no usable output (exit code 2)",
12748            ),
12749            (
12750                "exited with None and no usable output",
12751                "process exited with no usable output (exit code unavailable)",
12752            ),
12753        ] {
12754            let previous = FailClass::Other(failure_signature(old));
12755            let current = FailClass::Other(failure_signature(new));
12756            assert_eq!(previous, current, "resumption retains the failure class");
12757            assert!(!should_hand_over(Some(&previous), &current));
12758        }
12759        assert_eq!(
12760            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12761            failure_signature("cannot open /var/x.txt: denied")
12762        );
12763        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12764    }
12765
12766    #[test]
12767    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12768        let other = FailClass::Other("x".into());
12769        assert!(should_hand_over(None, &FailClass::Quota));
12770        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12771        assert!(should_hand_over(
12772            Some(&FailClass::Timeout),
12773            &FailClass::Timeout
12774        ));
12775        assert!(should_hand_over(None, &other));
12776        assert!(!should_hand_over(Some(&other), &other));
12777        assert!(should_hand_over(
12778            Some(&other),
12779            &FailClass::Other("y".into())
12780        ));
12781        // A quota or timeout in between ends the run of identical failures.
12782        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12783        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12784    }
12785
12786    #[test]
12787    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12788        let a = SeatState::new("judge-1", "alpha", 7);
12789        let b = handover_seat("judge-1", "beta", 7);
12790        assert_ne!(a.claude_session, b.claude_session);
12791        assert_eq!(b.turns, 0);
12792    }
12793
12794    #[test]
12795    fn a_timeout_is_classified_apart_from_other_failures() {
12796        assert_eq!(
12797            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12798            Some(FailClass::Timeout)
12799        );
12800        assert!(matches!(
12801            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12802            Some(FailClass::Other(_))
12803        ));
12804    }
12805
12806    #[test]
12807    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12808        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12809        let tried = BTreeSet::from(["beta".to_owned()]);
12810        // beta sits at index 1; the next candidate is gamma, never alpha —
12811        // which is very likely a different candidate slot's own agent.
12812        let next = next_untried_in_roster(&roster, 1, &tried);
12813        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12814    }
12815
12816    #[test]
12817    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12818        let roster = vec![spec("alpha"), spec("beta")];
12819        let tried = BTreeSet::from(["beta".to_owned()]);
12820        // beta is the roster's last entry: nothing follows it, and alpha —
12821        // earlier in the roster, almost certainly a different candidate
12822        // slot's own agent — must not be reached by wrapping back to it.
12823        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12824    }
12825
12826    #[test]
12827    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12828        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12829        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12830        // beta (index 1) and gamma (index 2, the only entry after it) have
12831        // both been tried; alpha (index 0) never has, but it comes before
12832        // beta's own position, so there is nothing further for this seat.
12833        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12834    }
12835
12836    #[test]
12837    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12838        let roster = vec![spec("a"), spec("a"), spec("b")];
12839        let tried = BTreeSet::from(["a".to_owned()]);
12840        let next = next_untried_in_roster(&roster, 0, &tried);
12841        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12842    }
12843
12844    #[test]
12845    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12846        let roster = vec![spec("a"), spec("b")];
12847        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12848        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12849    }
12850
12851    #[test]
12852    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12853        let dir = tempfile::tempdir().unwrap();
12854        let bay = dir.path().join("ffff");
12855
12856        // Not there yet: nothing to do, nothing to panic on.
12857        remove_if_empty(&bay);
12858        assert!(!bay.exists());
12859
12860        // Something still inside - the winner's worktree, or a stray file -
12861        // keeps the directory standing.
12862        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12863        remove_if_empty(&bay);
12864        assert!(bay.exists(), "non-empty directory must survive");
12865
12866        // Once the last entry is gone, so is the directory itself.
12867        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12868        remove_if_empty(&bay);
12869        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12870    }
12871
12872    // `round_is_clean` is the exact decision this task fixed: a round with a
12873    // seat that never answered must not read the same as a round every seat
12874    // actually reviewed. These are deterministic and process-free by design —
12875    // the equivalent end-to-end check (a real reviewer timing out under a
12876    // live graph run) is a genuine race against wall-clock contention, and a
12877    // spawn slow enough to blow even a generous budget under a loaded test
12878    // run must not turn this specific regression check flaky.
12879
12880    #[test]
12881    fn a_full_panel_that_found_nothing_is_clean() {
12882        assert!(round_is_clean(
12883            0,
12884            true,
12885            2,
12886            2,
12887            0,
12888            IncompleteReviewPolicy::Block
12889        ));
12890    }
12891
12892    #[test]
12893    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12894        assert!(!round_is_clean(
12895            0,
12896            true,
12897            1,
12898            2,
12899            0,
12900            IncompleteReviewPolicy::Block
12901        ));
12902    }
12903
12904    #[test]
12905    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12906        assert!(!round_is_clean(
12907            1,
12908            true,
12909            1,
12910            2,
12911            0,
12912            IncompleteReviewPolicy::Warn
12913        ));
12914    }
12915
12916    #[test]
12917    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12918        assert!(round_is_clean(
12919            0,
12920            true,
12921            1,
12922            2,
12923            0,
12924            IncompleteReviewPolicy::Warn
12925        ));
12926    }
12927
12928    #[test]
12929    fn a_full_panel_with_an_open_finding_is_not_clean() {
12930        assert!(!round_is_clean(
12931            1,
12932            true,
12933            2,
12934            2,
12935            0,
12936            IncompleteReviewPolicy::Block
12937        ));
12938    }
12939
12940    #[test]
12941    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12942        assert!(!round_is_clean(
12943            0,
12944            false,
12945            2,
12946            2,
12947            0,
12948            IncompleteReviewPolicy::Block
12949        ));
12950    }
12951
12952    // The stall this task closes: under the default `block` policy, a seat
12953    // missing only because it was rate limited must not force a wait for a
12954    // session limit that will not lift by the next round. `round_is_clean`
12955    // is where that quorum carve-out lives; the review loop around it never
12956    // changes what a reviewer's vote or a finding's severity means.
12957
12958    #[test]
12959    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12960        // 1 of 2 answered, and the one missing was quota'd — the exact
12961        // "review-2 rate limited (quota)" shape from the field report.
12962        assert!(round_is_clean(
12963            0,
12964            true,
12965            1,
12966            2,
12967            1,
12968            IncompleteReviewPolicy::Block
12969        ));
12970    }
12971
12972    #[test]
12973    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12974        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12975        // not a quota loss (`quota_missing` stays 0) — worth another try.
12976        assert!(!round_is_clean(
12977            0,
12978            true,
12979            1,
12980            2,
12981            0,
12982            IncompleteReviewPolicy::Block
12983        ));
12984    }
12985
12986    #[test]
12987    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12988        assert!(!round_is_clean(
12989            1,
12990            true,
12991            1,
12992            2,
12993            1,
12994            IncompleteReviewPolicy::Block
12995        ));
12996        assert!(!round_is_clean(
12997            0,
12998            false,
12999            1,
13000            2,
13001            1,
13002            IncompleteReviewPolicy::Block
13003        ));
13004    }
13005
13006    #[test]
13007    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13008        // Every seat quota'd, nobody answered: there is no panel to decide
13009        // on, so this must fall through to the existing block-and-retry
13010        // fallback rather than call an unreviewed patch clean.
13011        assert!(!round_is_clean(
13012            0,
13013            true,
13014            0,
13015            2,
13016            2,
13017            IncompleteReviewPolicy::Block
13018        ));
13019    }
13020
13021    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13022        CommandOutcome {
13023            command: "test".to_owned(),
13024            code,
13025            output_tail: String::new(),
13026            duration_ms: 0,
13027            resource_blocked,
13028        }
13029    }
13030
13031    #[test]
13032    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13033        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13034        assert!(
13035            !verify_inconclusive(&[outcome(Some(1), false)]),
13036            "an ordinary failure is still evidence about the patch"
13037        );
13038        assert!(verify_inconclusive(&[outcome(None, true)]));
13039        assert!(
13040            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13041            "one inconclusive outcome taints the whole batch"
13042        );
13043        assert!(!verify_inconclusive(&[]));
13044    }
13045
13046    #[tokio::test]
13047    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13048        // Alive for the first two checks, then dead - confirms the loop
13049        // actually re-polls rather than deciding once and sleeping out the
13050        // ceiling regardless.
13051        let calls = std::sync::atomic::AtomicUsize::new(0);
13052        let started = Instant::now();
13053        wait_for_pids_with(
13054            &[123],
13055            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13056            Duration::from_millis(5),
13057            Duration::from_secs(5),
13058        )
13059        .await;
13060        assert!(
13061            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13062            "must keep checking rather than deciding on the first answer"
13063        );
13064        assert!(
13065            started.elapsed() < Duration::from_secs(1),
13066            "must return the moment it is confirmed dead, not wait out the ceiling"
13067        );
13068    }
13069
13070    #[tokio::test]
13071    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13072        let started = Instant::now();
13073        wait_for_pids_with(
13074            &[123],
13075            |_| true, // never reports dead
13076            Duration::from_millis(5),
13077            Duration::from_millis(30),
13078        )
13079        .await;
13080        let elapsed = started.elapsed();
13081        assert!(
13082            elapsed >= Duration::from_millis(30),
13083            "must not give up before its own ceiling: {elapsed:?}"
13084        );
13085        assert!(
13086            elapsed < Duration::from_secs(1),
13087            "must not wait past its own ceiling either: {elapsed:?}"
13088        );
13089    }
13090
13091    #[tokio::test]
13092    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13093        let started = Instant::now();
13094        wait_for_pids_with(
13095            &[],
13096            |_| true,
13097            Duration::from_secs(5),
13098            Duration::from_secs(5),
13099        )
13100        .await;
13101        assert!(
13102            started.elapsed() < Duration::from_millis(200),
13103            "an empty pid list has nothing to confirm"
13104        );
13105    }
13106
13107    // `review_conclusion` is the exact decision the review hand-off task
13108    // fixed: a round budget spent (or a tree that stopped moving) must not
13109    // collapse into `Blocked` regardless of what verification actually
13110    // said. Deterministic and process-free for the same reason the
13111    // `round_is_clean` family above is.
13112    fn review_round(
13113        clean: bool,
13114        blocking: usize,
13115        answered: usize,
13116        expected: usize,
13117        progressed: bool,
13118        e2e_ok: bool,
13119    ) -> ReviewRound {
13120        ReviewRound {
13121            round: 1,
13122            head: "h".to_owned(),
13123            verified_head: None,
13124            verified_at: None,
13125            reviews: Vec::new(),
13126            e2e: vec![CommandOutcome {
13127                command: "test".to_owned(),
13128                code: Some(if e2e_ok { 0 } else { 1 }),
13129                output_tail: String::new(),
13130                duration_ms: 0,
13131                resource_blocked: false,
13132            }],
13133            verify_retried: false,
13134            e2e_deferred: false,
13135            e2e_defer_reason: None,
13136            fix: None,
13137            blocking,
13138            answered,
13139            expected,
13140            clean,
13141            progressed,
13142            vote_split: false,
13143            reconsideration: Vec::new(),
13144            verdict: None,
13145        }
13146    }
13147
13148    #[test]
13149    fn review_conclusion_is_none_when_nothing_has_run() {
13150        assert_eq!(review_conclusion(&[], 3), None);
13151    }
13152
13153    #[test]
13154    fn review_conclusion_is_none_while_rounds_remain() {
13155        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13156        assert_eq!(review_conclusion(&rounds, 3), None);
13157    }
13158
13159    #[test]
13160    fn review_conclusion_is_gating_once_a_round_is_clean() {
13161        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13162        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13163    }
13164
13165    #[test]
13166    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13167        let rounds = vec![
13168            review_round(false, 1, 2, 2, true, true),
13169            review_round(false, 1, 2, 2, true, true),
13170        ];
13171        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13172    }
13173
13174    #[test]
13175    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13176        let rounds = vec![
13177            review_round(false, 1, 2, 2, true, true),
13178            review_round(false, 1, 2, 2, true, false),
13179        ];
13180        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13181    }
13182
13183    #[test]
13184    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13185        // Magi never got a command to run against this round's own head — a
13186        // resource-blocked attempt, not a red one — so this must never
13187        // settle on `Blocked` the way a genuine e2e failure would. `None`
13188        // here is what tells `Runner::review_loop` to retry the check
13189        // itself rather than trust this cheap recomputation with a verdict
13190        // it cannot actually produce.
13191        let mut blocked = review_round(false, 1, 2, 2, true, false);
13192        blocked.e2e[0].resource_blocked = true;
13193        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13194        assert_eq!(review_conclusion(&rounds, 2), None);
13195    }
13196
13197    #[test]
13198    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13199        // Missing input, not a verified tree — never a hand-off candidate.
13200        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13201        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13202    }
13203
13204    #[test]
13205    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13206        let rounds = vec![
13207            review_round(false, 1, 2, 2, false, true),
13208            review_round(false, 1, 2, 2, false, true),
13209        ];
13210        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13211    }
13212
13213    fn secs(n: u64) -> Duration {
13214        Duration::from_secs(n)
13215    }
13216
13217    /// A throwaway repo with one commit on `main`, for tests that need `merge`
13218    /// to make real (and, if it runs at all, real*ly fail*) git calls.
13219    #[tokio::test]
13220    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13221        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13222        if std::env::var_os(CHILD).is_none() {
13223            // Other tests reconcile every run in the process-global home,
13224            // briefly taking owner locks. This ancestry test expects an idle
13225            // predecessor, so give it its own process and pinned test home.
13226            let output = std::process::Command::new(std::env::current_exe().unwrap())
13227                .args([
13228                    "--exact",
13229                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13230                    "--nocapture",
13231                ])
13232                .env(CHILD, "1")
13233                .env_remove("MAGI_WORKER_HOME")
13234                .env_remove("MAGI_WORKER_ID")
13235                .output()
13236                .unwrap();
13237            assert!(
13238                output.status.success()
13239                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13240                "{}\n{}",
13241                String::from_utf8_lossy(&output.stdout),
13242                String::from_utf8_lossy(&output.stderr)
13243            );
13244            return;
13245        }
13246        ask_test_home();
13247        let tmp = tempfile::tempdir().unwrap();
13248        let repo = tmp.path().join("repo");
13249        std::fs::create_dir_all(&repo).unwrap();
13250        init_repo(&repo);
13251        let g = |args: &[&str]| {
13252            let out = std::process::Command::new("git")
13253                .args(args)
13254                .current_dir(&repo)
13255                .quiet()
13256                .output()
13257                .unwrap();
13258            assert!(
13259                out.status.success(),
13260                "{}",
13261                String::from_utf8_lossy(&out.stderr)
13262            );
13263        };
13264        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13265        g(&["checkout", "-b", "historical"]);
13266        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13267        g(&["add", "."]);
13268        g(&["commit", "-m", "old alternative"]);
13269        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13270        g(&["checkout", "-b", "magi/task-selected", &base]);
13271        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13272        g(&["add", "."]);
13273        g(&["commit", "-m", "measurement infrastructure"]);
13274        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13275        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13276        g(&["add", "."]);
13277        g(&["commit", "-m", "dependent implementation"]);
13278        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13279        g(&["checkout", "main"]);
13280        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13281        g(&["add", "."]);
13282        g(&["commit", "-m", "unrelated base change"]);
13283        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13284        let origin = tmp.path().join("origin.git");
13285        g(&[
13286            "clone",
13287            "--bare",
13288            repo.to_str().unwrap(),
13289            origin.to_str().unwrap(),
13290        ]);
13291        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13292        g(&["fetch", "origin"]);
13293        let selection = crate::pr_recovery::Selection {
13294            url: "https://example.test/project/pull/1".into(),
13295            branch: "magi/task-selected".into(),
13296            head: head.clone(),
13297            source_head: None,
13298            source_run: None,
13299            source_proof: None,
13300            disposition: Some(crate::pr_recovery::Disposition::Review),
13301            instruction: "preserve measured behavior".into(),
13302            approved_at: jiff::Timestamp::now(),
13303        };
13304        let mut template = runner_at(RunStatus::Reviewing);
13305        template.state.repo = repo.clone();
13306        template.state.base_commit = current.clone();
13307        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13308        template.state.origin = Some(crate::run::Origin::queue("selected"));
13309        template.state.pr_head = Some(selection.branch.clone());
13310        template.state.pr_recovery = Some(selection.clone());
13311        let mut runner =
13312            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13313                .await
13314                .unwrap();
13315        let wt = runner.state.winner().unwrap().worktree.clone();
13316        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13317        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13318        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13319        assert!(runner.state.seeds.is_empty());
13320        assert_eq!(runner.state.instruction, "preserve measured behavior");
13321        runner.sync_to_base().await.unwrap();
13322        assert_eq!(
13323            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13324            "prerequisite"
13325        );
13326        assert_eq!(
13327            std::fs::read_to_string(wt.join("implementation")).unwrap(),
13328            "measured implementation"
13329        );
13330        assert_eq!(
13331            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13332            "current main change"
13333        );
13334        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13335        assert_eq!(
13336            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13337            head
13338        );
13339
13340        // A failed selected-source setup stays inspectable, never resumable as a new competition.
13341        let mut partial = runner_at(RunStatus::Reviewing);
13342        partial.state.repo = repo.clone();
13343        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13344        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13345            run: Some("exit 1".into()),
13346            ..Default::default()
13347        }];
13348        let selection = runner.state.pr_recovery.clone().unwrap();
13349        partial.state.pr_recovery = Some(selection.clone());
13350        partial.state.pr_head = Some(selection.branch.clone());
13351        partial.state.origin = Some(crate::run::Origin::queue("selected"));
13352        let id = partial.state.id.clone();
13353        let error = match Runner::open_selected(
13354            &repo,
13355            partial.state,
13356            partial.roles,
13357            1,
13358            runner.state.base_commit.clone(),
13359            selection,
13360        )
13361        .await
13362        {
13363            Ok(_) => panic!("setup must fail"),
13364            Err(e) => e,
13365        };
13366        assert_eq!(
13367            error
13368                .downcast_ref::<crate::pr_recovery::StartupFailed>()
13369                .unwrap()
13370                .run,
13371            id
13372        );
13373        let failed = RunState::load(&id).unwrap();
13374        assert_eq!(failed.status, RunStatus::Blocked);
13375        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13376        assert!(Runner::resume(&id).is_err());
13377        runner.state.status = RunStatus::Ready;
13378        runner.state.driver_exited = true;
13379        runner.state.save().unwrap();
13380        let mut retry = runner_at(RunStatus::Reviewing);
13381        retry.state.repo = repo.clone();
13382        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13383        let mut reselected = runner.state.pr_recovery.clone().unwrap();
13384        reselected.approved_at = jiff::Timestamp::now();
13385        retry.state.origin = Some(crate::run::Origin::queue("selected"));
13386        retry.state.pr_head = Some(reselected.branch.clone());
13387        retry.state.pr_recovery = Some(reselected.clone());
13388        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13389            .await
13390            .unwrap();
13391        let restarted = Runner::open_selected(
13392            &repo,
13393            retry.state,
13394            retry.roles,
13395            1,
13396            runner.state.base_commit.clone(),
13397            reselected,
13398        )
13399        .await
13400        .unwrap();
13401        assert_eq!(
13402            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13403                .await
13404                .unwrap(),
13405            head
13406        );
13407        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13408        assert_eq!(
13409            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13410            head
13411        );
13412    }
13413
13414    #[tokio::test]
13415    #[cfg(unix)]
13416    async fn fallback_and_interrupted_restoration_require_coordination() {
13417        use std::os::fd::AsRawFd as _;
13418        ask_test_home();
13419        let tmp = tempfile::tempdir().unwrap();
13420        let mut runner = registered_runner(tmp.path()).await;
13421        runner.state.config.graph.timeout_resource_wait = 1;
13422        let winner = runner.state.candidates[0].clone();
13423        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13424        let leftover = winner.worktree.join("leftover");
13425        std::fs::write(&leftover, "untrusted edits").unwrap();
13426        let next = tmp.path().join("next-fixer");
13427        let mut failed = spec("first");
13428        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13429        let mut successor = spec("second");
13430        successor.command = vec![
13431            "sh".into(),
13432            "-c".into(),
13433            format!("touch '{}'; echo done", next.display()),
13434        ];
13435        let attempts = vec![
13436            (failed.clone(), "first".into()),
13437            (successor.clone(), "second".into()),
13438        ];
13439        let lock_path = runner.state.repo.join(".git/updater.lock");
13440        let external = std::fs::File::create(&lock_path).unwrap();
13441        assert_eq!(
13442            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13443            0
13444        );
13445        let artifacts = tmp.path().join("artifacts");
13446        let build = |spec, seat| SeatJob {
13447            spec,
13448            seat,
13449            cwd: winner.worktree.clone(),
13450            prompt: "fixture".into(),
13451            timeout: Duration::from_secs(10),
13452            allow_write: true,
13453            sessions: false,
13454            artifacts: artifacts.clone(),
13455            stem: "fix".into(),
13456            handover: None,
13457        };
13458        let (_, _, out) = runner
13459            .ask_fixer_among(
13460                attempts.clone(),
13461                "graft",
13462                None,
13463                Some((&winner.worktree, &head)),
13464                build,
13465            )
13466            .await;
13467        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13468        assert!(leftover.exists(), "contention must prevent clean");
13469        assert!(!next.exists());
13470        assert!(runner.state.handovers.is_empty());
13471        // Early interrupted-repair rollback uses the same protected route.
13472        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13473            from_run: "earlier".into(),
13474            generation: 1,
13475            mode: crate::recovery::Mode::VerifyRepair,
13476            reasons: Vec::new(),
13477            reviewed_head: None,
13478            rounds: Vec::new(),
13479            open_findings: Vec::new(),
13480            evidence: None,
13481            criteria_inherited: true,
13482            repair: crate::recovery::Repair::Started {
13483                before: head.clone(),
13484            },
13485        });
13486        assert!(
13487            runner
13488                .recovery_repair(&winner)
13489                .await
13490                .unwrap_err()
13491                .is::<crate::repo_lock::ResourceBlocked>()
13492        );
13493        assert!(matches!(
13494            runner.state.recovery.as_ref().unwrap().repair,
13495            crate::recovery::Repair::Started { .. }
13496        ));
13497        assert!(leftover.exists());
13498        drop(external);
13499        runner.recovery_repair(&winner).await.unwrap();
13500        assert!(!leftover.exists());
13501        assert!(matches!(
13502            runner.state.recovery.as_ref().unwrap().repair,
13503            crate::recovery::Repair::Done { .. }
13504        ));
13505        // A hard lock-open failure also leaves HEAD and untracked work intact.
13506        std::fs::remove_file(&lock_path).unwrap();
13507        std::fs::create_dir(&lock_path).unwrap();
13508        std::fs::write(&leftover, "untrusted edits").unwrap();
13509        let (_, _, out) = runner
13510            .ask_fixer_among(
13511                attempts,
13512                "graft",
13513                None,
13514                Some((&winner.worktree, &head)),
13515                build,
13516            )
13517            .await;
13518        assert!(matches!(out, AgentOutcome::Failed(_)));
13519        assert!(leftover.exists());
13520        assert!(!next.exists());
13521        assert_eq!(
13522            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13523            head
13524        );
13525    }
13526
13527    #[cfg(unix)]
13528    async fn registered_runner(root: &Path) -> Runner {
13529        ask_test_home();
13530        let repo = root.join("repo");
13531        std::fs::create_dir_all(&repo).unwrap();
13532        init_repo(&repo);
13533        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13534        let mut runner = runner_at(RunStatus::Prep);
13535        runner.state.repo = repo.clone();
13536        runner.state.config.coordination = crate::config::test_coordination();
13537        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13538        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13539        runner.state.config.graph.advise = false;
13540        runner.roles.implementers = vec![spec("implement")];
13541        runner.roles.judges = vec![spec("judge")];
13542        runner.prep().await.unwrap();
13543        runner
13544    }
13545
13546    #[tokio::test]
13547    #[cfg(unix)]
13548    async fn parked_implementer_persists_completed_work() {
13549        for registered in [false, true] {
13550            let tmp = tempfile::tempdir().unwrap();
13551            let mut runner = registered_runner(tmp.path()).await;
13552            if !registered {
13553                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13554            }
13555            let marker = tmp.path().join("started");
13556            let release = tmp.path().join("release");
13557            let mut implementer = spec("implement");
13558            implementer.command = vec![
13559                "sh".into(),
13560                "-c".into(),
13561                format!(
13562                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13563                    marker.display(),
13564                    release.display()
13565                ),
13566            ];
13567            runner.roles.implementers = vec![implementer.clone()];
13568            runner.state.config.agents = vec![implementer];
13569            let pause = runner.pause.clone();
13570            let (result, ()) = tokio::join!(runner.implement(), async {
13571                tokio::time::timeout(Duration::from_secs(10), async {
13572                    while !marker.exists() {
13573                        tokio::time::sleep(Duration::from_millis(10)).await;
13574                    }
13575                })
13576                .await
13577                .unwrap();
13578                pause.park();
13579                std::fs::write(&release, "go").unwrap();
13580            });
13581            result.unwrap();
13582            let candidate = &runner.state.candidates[0];
13583            assert!(candidate.commits > 0, "{candidate:?}");
13584            assert!(candidate.summary.contains("saved-result"));
13585            assert_eq!(candidate.files, 1);
13586            let before = candidate.commits;
13587            runner.implement().await.unwrap();
13588            assert_eq!(runner.state.candidates[0].commits, before);
13589        }
13590    }
13591
13592    #[tokio::test]
13593    #[cfg(unix)]
13594    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13595        let tmp = tempfile::tempdir().unwrap();
13596        let mut runner = registered_runner(tmp.path()).await;
13597        let winner = runner.state.candidates[0].clone();
13598        let cache = tmp.path().join("cache");
13599        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13600        runner.state.config.graph.timeout_resource_wait = 0;
13601        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13602        let crate::cache::AcquireOutcome::Acquired(held) =
13603            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13604        else {
13605            panic!("cache must be free")
13606        };
13607        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13608            from_run: "earlier".into(),
13609            generation: 1,
13610            mode: crate::recovery::Mode::VerifyRepair,
13611            reasons: Vec::new(),
13612            reviewed_head: None,
13613            rounds: Vec::new(),
13614            open_findings: Vec::new(),
13615            evidence: None,
13616            criteria_inherited: true,
13617            repair: crate::recovery::Repair::NotStarted,
13618        });
13619        runner.recovery_repair(&winner).await.unwrap();
13620        assert!(matches!(
13621            runner.state.recovery.as_ref().unwrap().repair,
13622            crate::recovery::Repair::WaitingForRestore { .. }
13623        ));
13624        drop(held);
13625        runner.state.resource_block = None;
13626        runner.state.config.graph.timeout_resource_wait = 1;
13627        runner.recovery_repair(&winner).await.unwrap();
13628        assert!(matches!(
13629            runner.state.recovery.as_ref().unwrap().repair,
13630            crate::recovery::Repair::Done { .. }
13631        ));
13632        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13633    }
13634
13635    #[tokio::test]
13636    #[cfg(unix)]
13637    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13638        use std::os::fd::AsRawFd as _;
13639        for graft in [false, true] {
13640            let tmp = tempfile::tempdir().unwrap();
13641            let mut runner = registered_runner(tmp.path()).await;
13642            runner.state.config.graph.timeout_resource_wait = 1;
13643            let winner = runner.state.candidates[0].clone();
13644            let marker = tmp.path().join("fixer-started");
13645            let mut fixer = spec("fixer");
13646            fixer.command = vec![
13647                "sh".into(),
13648                "-c".into(),
13649                format!("echo started >> '{}'; echo done", marker.display()),
13650            ];
13651            runner.roles.fixer = Some(vec![fixer]);
13652            if graft {
13653                let mut loser = winner.clone();
13654                loser.label = 'B';
13655                loser.index = 1;
13656                runner.state.candidates.push(loser);
13657                runner.state.config.graph.graft_runner_up = true;
13658                runner.state.tally = Some(Tally {
13659                    first_choice: BTreeMap::new(),
13660                    borda: BTreeMap::new(),
13661                    winner: winner.label,
13662                    rankings: 1,
13663                    unanimous_initial: true,
13664                    deliberated: false,
13665                    changed_votes: 0,
13666                    unanimous_final: true,
13667                    tie_break: None,
13668                    judges: 1,
13669                    present: 1,
13670                    quorum: 1,
13671                    met_quorum: true,
13672                    uncontested: None,
13673                });
13674                runner.state.judgements.push(crate::run::Judgement {
13675                    judge: 1,
13676                    seat: "judge-1".into(),
13677                    agent: "judge".into(),
13678                    ranking: vec![winner.label, 'B'],
13679                    reasons: BTreeMap::new(),
13680                    confidence: None,
13681                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13682                    order: vec![0, 1],
13683                    failed: None,
13684                    duration_ms: 0,
13685                });
13686            } else {
13687                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13688                    from_run: "earlier".into(),
13689                    generation: 1,
13690                    mode: crate::recovery::Mode::VerifyRepair,
13691                    reasons: Vec::new(),
13692                    reviewed_head: None,
13693                    rounds: Vec::new(),
13694                    open_findings: Vec::new(),
13695                    evidence: None,
13696                    criteria_inherited: true,
13697                    repair: crate::recovery::Repair::NotStarted,
13698                });
13699            }
13700            let reader = crate::repo_lock::SeatLease::for_checkout(
13701                &winner.worktree,
13702                false,
13703                crate::config::test_coordination().as_ref(),
13704            )
13705            .await
13706            .unwrap();
13707            let pause = runner.pause.clone();
13708            let (result, ()) = tokio::join!(
13709                async {
13710                    if graft {
13711                        runner.graft().await
13712                    } else {
13713                        runner.recovery_repair(&winner).await
13714                    }
13715                },
13716                async {
13717                    tokio::time::sleep(Duration::from_millis(100)).await;
13718                    pause.park();
13719                }
13720            );
13721            result.unwrap();
13722            assert!(!marker.exists(), "a blocked seat must not spawn");
13723            // Exercise the persisted state, as a binary replacement/resume does.
13724            runner.state = RunState::load(&runner.state.id).unwrap();
13725            if graft {
13726                assert_eq!(
13727                    runner.state.graft.as_ref().unwrap().outcome,
13728                    GraftOutcome::WaitingForRestore
13729                );
13730            } else {
13731                assert!(matches!(
13732                    runner.state.recovery.as_ref().unwrap().repair,
13733                    crate::recovery::Repair::WaitingForRestore { .. }
13734                ));
13735            }
13736            drop(reader);
13737            runner.pause = Pause::new();
13738            runner.state.resource_block = None;
13739            // Deferred restoration must still respect an external lock owner.
13740            let external =
13741                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13742            assert_eq!(
13743                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13744                0
13745            );
13746            let leftover = winner.worktree.join("leftover");
13747            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13748            let result = if graft {
13749                runner.graft().await
13750            } else {
13751                runner.recovery_repair(&winner).await
13752            };
13753            assert!(
13754                result
13755                    .unwrap_err()
13756                    .is::<crate::repo_lock::ResourceBlocked>()
13757            );
13758            assert!(leftover.exists());
13759            assert!(!marker.exists());
13760            drop(external);
13761            runner.state.resource_block = None;
13762            let result = if graft {
13763                runner.graft().await
13764            } else {
13765                runner.recovery_repair(&winner).await
13766            };
13767            result.unwrap();
13768            assert!(!leftover.exists(), "restore must precede retry");
13769            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13770            if graft {
13771                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13772            } else {
13773                assert!(matches!(
13774                    runner.state.recovery.as_ref().unwrap().repair,
13775                    crate::recovery::Repair::Done { .. }
13776                ));
13777            }
13778        }
13779    }
13780
13781    #[tokio::test]
13782    #[cfg(unix)]
13783    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13784        use std::os::fd::AsRawFd as _;
13785        let tmp = tempfile::tempdir().unwrap();
13786        let mut runner = registered_runner(tmp.path()).await;
13787        runner.state.config.graph.timeout_resource_wait = 1;
13788        let winner = runner.state.candidates[0].clone();
13789        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13790        runner.state.tally = Some(Tally {
13791            first_choice: BTreeMap::new(),
13792            borda: BTreeMap::new(),
13793            winner: winner.label,
13794            rankings: 0,
13795            unanimous_initial: false,
13796            deliberated: false,
13797            changed_votes: 0,
13798            unanimous_final: false,
13799            tie_break: None,
13800            judges: 0,
13801            present: 0,
13802            quorum: 0,
13803            met_quorum: true,
13804            uncontested: Some("fixture".into()),
13805        });
13806        runner.state.graft = Some(GraftRecord {
13807            pre: pre.clone(),
13808            post: None,
13809            lines: None,
13810            limit: None,
13811            agent: None,
13812            outcome: GraftOutcome::Running,
13813            requested: Vec::new(),
13814            adoption: None,
13815            adoption_error: None,
13816        });
13817        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13818        git::commit_all(&winner.worktree, "unfinished graft")
13819            .await
13820            .unwrap();
13821        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13822        let leftover = winner.worktree.join("leftover");
13823        std::fs::write(&leftover, "untrusted edits").unwrap();
13824        let path = runner.state.repo.join(".git/updater.lock");
13825        let external = std::fs::File::create(&path).unwrap();
13826        assert_eq!(
13827            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13828            0
13829        );
13830        assert!(
13831            runner
13832                .graft()
13833                .await
13834                .unwrap_err()
13835                .is::<crate::repo_lock::ResourceBlocked>()
13836        );
13837        assert!(matches!(
13838            runner.state.graft.as_ref().unwrap().outcome,
13839            GraftOutcome::Running
13840        ));
13841        assert_eq!(
13842            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13843            unfinished
13844        );
13845        assert!(leftover.exists());
13846        drop(external);
13847        std::fs::remove_file(&path).unwrap();
13848        std::fs::create_dir(&path).unwrap();
13849        assert!(runner.graft().await.is_err());
13850        assert!(matches!(
13851            runner.state.graft.as_ref().unwrap().outcome,
13852            GraftOutcome::Running
13853        ));
13854        assert_eq!(
13855            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13856            unfinished
13857        );
13858        assert!(leftover.exists());
13859        std::fs::remove_dir(&path).unwrap();
13860        runner.graft().await.unwrap();
13861        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13862        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13863        assert!(!leftover.exists());
13864    }
13865
13866    #[tokio::test]
13867    #[cfg(unix)]
13868    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13869        use std::os::fd::AsRawFd as _;
13870        let tmp = tempfile::tempdir().unwrap();
13871        let mut runner = registered_runner(tmp.path()).await;
13872        runner.state.prep_complete = Some(false);
13873        runner.state.config.graph.timeout_resource_wait = 1;
13874        runner.state.save().unwrap();
13875        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13876        assert_eq!(
13877            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13878            0
13879        );
13880        runner.execute().await.unwrap();
13881        let disk = RunState::load(&runner.state.id).unwrap();
13882        assert!(disk.driver_exited);
13883        assert_eq!(disk.driver_pid, None);
13884        assert_eq!(runner.state.status, RunStatus::Blocked);
13885        assert_eq!(
13886            runner.state.resource_block.as_ref().unwrap().kind,
13887            "repository coordination"
13888        );
13889        assert!(runner.state.prep_failure.is_none());
13890        assert!(runner.state.seats.is_empty());
13891        runner.pause.park();
13892        // Include headroom for Git subprocesses under parallel test load;
13893        // the repository contention deadline itself remains one second.
13894        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13895            .await
13896            .unwrap()
13897            .unwrap();
13898        assert!(runner.state.parked);
13899        assert!(runner.state.prep_failure.is_none());
13900        let disk = RunState::load(&runner.state.id).unwrap();
13901        assert!(disk.driver_exited);
13902        assert_eq!(disk.driver_pid, None);
13903    }
13904
13905    #[tokio::test]
13906    #[cfg(unix)]
13907    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13908        use std::os::fd::AsRawFd as _;
13909        let tmp = tempfile::tempdir().unwrap();
13910        let mut runner = registered_runner(tmp.path()).await;
13911        runner.state.prep_complete = Some(false);
13912        runner.state.config.graph.timeout_resource_wait = 30;
13913        runner.state.save().unwrap();
13914        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13915        assert_eq!(
13916            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13917            0
13918        );
13919        assert!(
13920            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13921                .await
13922                .is_err()
13923        );
13924        let disk = RunState::load(&runner.state.id).unwrap();
13925        assert!(disk.driver_exited);
13926        assert_eq!(disk.driver_pid, None);
13927        assert!(disk.status.resumable());
13928        assert!(runner.state.driver_exited);
13929        assert_eq!(runner.state.driver_pid, None);
13930        runner.state.save().unwrap();
13931        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13932    }
13933
13934    #[tokio::test]
13935    #[cfg(unix)]
13936    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13937        let tmp = tempfile::tempdir().unwrap();
13938        let mut runner = registered_runner(tmp.path()).await;
13939        let winner = runner.state.candidates[0].clone();
13940        let cache = tmp.path().join("cache");
13941        let started = tmp.path().join("pre-gate-started");
13942        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13943        runner.state.config.verify.gate =
13944            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13945        let home = crate::run::home();
13946        let owner =
13947            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13948        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13949            crate::cache::AcquireOutcome::Acquired(g) => g,
13950            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13951        };
13952        let mut gate = Box::pin(runner.run_gate(&winner));
13953        assert!(
13954            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13955                .await
13956                .is_err()
13957        );
13958        assert!(!started.exists(), "pre-gate must wait for the cache too");
13959        // Simulate the wave that already owns the cache entering invoke.
13960        // It must obtain the editor reservation and finish without waiting
13961        // for the gate that is queued behind its cache.
13962        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13963        let editor = crate::repo_lock::SeatLease::checkout_with(
13964            &winner.worktree,
13965            true,
13966            crate::config::test_coordination().as_ref(),
13967            &wait,
13968        )
13969        .await
13970        .unwrap();
13971        drop(editor);
13972        guard.release();
13973        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13974            .await
13975            .unwrap()
13976            .unwrap();
13977        drop(gate);
13978        assert!(outcomes.iter().all(CommandOutcome::ok));
13979        assert!(started.exists());
13980    }
13981
13982    #[tokio::test]
13983    #[cfg(unix)]
13984    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13985        use std::os::fd::AsRawFd as _;
13986
13987        ask_test_home();
13988        let tmp = tempfile::tempdir().unwrap();
13989        let repo = tmp.path().join("repo");
13990        std::fs::create_dir_all(&repo).unwrap();
13991        init_repo(&repo);
13992        let common = repo.join(".git");
13993        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13994        let mut runner = runner_at(RunStatus::Prep);
13995        runner.state.repo = repo.clone();
13996        runner.state.config.coordination = crate::config::test_coordination();
13997        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13998        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13999        runner.state.config.graph.advise = false;
14000        runner.roles.implementers = vec![spec("implement")];
14001        runner.roles.judges = vec![spec("judge")];
14002        runner.prep().await.unwrap();
14003        let winner = runner.state.candidates[0].clone();
14004        // Emulate edits already left by a completed seat. The fixture CLI
14005        // only reports completion; it performs no repository operations.
14006        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14007        let completed = tmp.path().join("seat-completed");
14008        runner.roles.implementers[0].command = vec![
14009            "sh".into(),
14010            "-c".into(),
14011            format!("touch '{}'; echo completed", completed.display()),
14012        ];
14013        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14014        assert_eq!(
14015            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14016            0
14017        );
14018        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14019        let mut implement = Box::pin(runner.implement());
14020        tokio::select! {
14021            result = &mut implement => panic!("rescue must wait: {result:?}"),
14022            result = tokio::time::timeout(Duration::from_secs(10), async {
14023                while !completed.exists() {
14024                    tokio::time::sleep(Duration::from_millis(20)).await;
14025                }
14026            }) => result.unwrap(),
14027        }
14028        assert!(
14029            tokio::time::timeout(Duration::from_millis(200), &mut implement)
14030                .await
14031                .is_err()
14032        );
14033        assert_eq!(
14034            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14035            before
14036        );
14037        assert_eq!(
14038            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14039            0
14040        );
14041        tokio::time::timeout(Duration::from_secs(10), &mut implement)
14042            .await
14043            .unwrap()
14044            .unwrap();
14045        drop(implement);
14046        assert_ne!(
14047            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14048            before
14049        );
14050        assert!(git::is_clean(&winner.worktree).await.unwrap());
14051
14052        let started = tmp.path().join("pre-gate-started");
14053        let proceed = tmp.path().join("pre-gate-proceed");
14054        let checked = tmp.path().join("gate-ran");
14055        runner.state.config.verify.pre_gate = vec![format!(
14056            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14057            started.display(),
14058            proceed.display()
14059        )];
14060        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14061        assert_eq!(
14062            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14063            0
14064        );
14065        let mut gate = Box::pin(runner.run_gate(&winner));
14066        assert!(
14067            tokio::time::timeout(Duration::from_millis(200), &mut gate)
14068                .await
14069                .is_err()
14070        );
14071        assert!(!started.exists());
14072        assert!(!checked.exists());
14073        assert!(git::is_clean(&winner.worktree).await.unwrap());
14074        assert_eq!(
14075            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14076            0
14077        );
14078        tokio::select! {
14079            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14080            result = tokio::time::timeout(Duration::from_secs(10), async {
14081                while !started.exists() {
14082                    tokio::time::sleep(Duration::from_millis(20)).await;
14083                }
14084            }) => result.unwrap(),
14085        }
14086        assert_eq!(
14087            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14088            -1,
14089            "mechanical edits must hold the real lock"
14090        );
14091        std::fs::write(proceed, "ready").unwrap();
14092        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14093            .await
14094            .unwrap()
14095            .unwrap();
14096        drop(gate);
14097        assert!(outcomes.iter().all(CommandOutcome::ok));
14098        assert!(checked.exists());
14099        assert!(runner.state.pre_gate_commit.is_some());
14100        assert!(git::is_clean(&winner.worktree).await.unwrap());
14101        assert_eq!(
14102            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14103            0,
14104            "gate completion must release coordination"
14105        );
14106        drop(outsider);
14107        std::fs::remove_file(common.join("updater.lock")).unwrap();
14108        std::fs::create_dir(common.join("updater.lock")).unwrap();
14109        std::fs::remove_file(&started).unwrap();
14110        std::fs::remove_file(&checked).unwrap();
14111        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14112        let error = runner.run_gate(&winner).await.unwrap_err();
14113        assert!(
14114            error
14115                .to_string()
14116                .contains("acquire mandatory repository lock")
14117        );
14118        assert!(
14119            !started.exists(),
14120            "failed acquisition must prevent pre-gate"
14121        );
14122        assert!(!checked.exists(), "failed acquisition must prevent gate");
14123        assert!(git::is_clean(&winner.worktree).await.unwrap());
14124        assert_eq!(
14125            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14126            head
14127        );
14128    }
14129
14130    #[tokio::test]
14131    #[cfg(unix)]
14132    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14133        use std::os::fd::AsRawFd as _;
14134        use std::time::Duration;
14135
14136        ask_test_home();
14137        let tmp = tempfile::tempdir().unwrap();
14138        let repo = tmp.path().join("repo");
14139        std::fs::create_dir_all(&repo).unwrap();
14140        init_repo(&repo);
14141        let common = repo.join(".git");
14142        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14143        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14144        assert_eq!(
14145            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14146            0
14147        );
14148        let mut runner = runner_at(RunStatus::Prep);
14149        runner.state.repo = repo.clone();
14150        runner.state.config.coordination = crate::config::test_coordination();
14151        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14152        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14153        runner.state.config.graph.advise = true;
14154        runner.state.config.graph.advisors = 1;
14155        runner.roles.implementers = vec![spec("implement")];
14156        runner.roles.judges = vec![spec("judge")];
14157        let root = runner.state.worktree_root();
14158        let started = tmp.path().join("setup-started");
14159        let proceed = tmp.path().join("setup-proceed");
14160        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14161            run: Some(format!(
14162                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14163                started.display(),
14164                proceed.display()
14165            )),
14166            ..Default::default()
14167        }];
14168        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14169        let config = git::git(&repo, &["config", "--local", "--list"])
14170            .await
14171            .unwrap();
14172        let mut prep = Box::pin(runner.prep());
14173        assert!(
14174            tokio::time::timeout(Duration::from_millis(200), &mut prep)
14175                .await
14176                .is_err()
14177        );
14178        assert!(
14179            !root.exists(),
14180            "contention must prevent even checkout creation"
14181        );
14182        assert!(!started.exists(), "contention must prevent setup");
14183        assert_eq!(
14184            git::git(&repo, &["branch", "--list"]).await.unwrap(),
14185            branches
14186        );
14187        assert_eq!(
14188            git::git(&repo, &["config", "--local", "--list"])
14189                .await
14190                .unwrap(),
14191            config
14192        );
14193        drop(lock);
14194        tokio::select! {
14195            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14196            result = tokio::time::timeout(Duration::from_secs(10), async {
14197                while !started.exists() {
14198                    tokio::time::sleep(Duration::from_millis(20)).await;
14199                }
14200            }) => result.unwrap(),
14201        }
14202        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14203        assert_eq!(
14204            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14205            -1
14206        );
14207        assert!(
14208            tokio::time::timeout(
14209                Duration::from_millis(100),
14210                crate::repo_lock::SeatLease::for_checkout(
14211                    &repo,
14212                    true,
14213                    crate::config::test_coordination().as_ref()
14214                )
14215            )
14216            .await
14217            .is_err(),
14218            "preparation must also exclude local editors"
14219        );
14220        std::fs::write(proceed, "ready").unwrap();
14221        tokio::time::timeout(Duration::from_secs(10), &mut prep)
14222            .await
14223            .unwrap()
14224            .unwrap();
14225        drop(prep);
14226        assert_eq!(runner.state.prep_complete, Some(true));
14227        assert!(root.join("judge-1").exists());
14228        assert!(root.join("advisor-1").exists());
14229        assert_eq!(
14230            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14231            0
14232        );
14233    }
14234
14235    #[tokio::test]
14236    #[cfg(unix)]
14237    async fn preparation_lock_failure_prevents_repository_changes() {
14238        ask_test_home();
14239        let tmp = tempfile::tempdir().unwrap();
14240        let repo = tmp.path().join("repo");
14241        std::fs::create_dir_all(&repo).unwrap();
14242        init_repo(&repo);
14243        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14244        // A directory at the required lock path makes opening the real lock
14245        // fail, rather than merely exercising an invalid registration.
14246        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14247        let mut runner = runner_at(RunStatus::Prep);
14248        runner.state.repo = repo.clone();
14249        runner.state.config.coordination = crate::config::test_coordination();
14250        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14251        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14252        runner.roles.implementers = vec![spec("implement")];
14253        let error = runner.prep().await.unwrap_err();
14254        assert!(
14255            error
14256                .to_string()
14257                .contains("acquire mandatory repository lock")
14258        );
14259        assert!(!runner.state.worktree_root().exists());
14260        assert!(runner.state.candidates.is_empty());
14261        assert!(!repo.join(".git/config.worktree").exists());
14262        assert_eq!(
14263            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14264            "* main"
14265        );
14266    }
14267
14268    #[tokio::test]
14269    async fn preparation_records_creation_intent_before_git_can_fail() {
14270        ask_test_home();
14271        let tmp = tempfile::tempdir().unwrap();
14272        let repo = tmp.path().join("repo");
14273        std::fs::create_dir_all(&repo).unwrap();
14274        init_repo(&repo);
14275        let mut runner = runner_at(RunStatus::Prep);
14276        runner.state.repo = repo.clone();
14277        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14278        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14279        runner.state.config.blind.commit_msg_hook = false;
14280        runner.roles.implementers = vec![spec("one")];
14281        let label = blind::assign_labels(1, runner.state.seed)[0];
14282        let branch = runner.state.branch_for(label);
14283        // A conflicting branch makes Git refuse creation before a candidate
14284        // can be registered. The durable intent must still identify it.
14285        git::git(&repo, &["branch", &branch]).await.unwrap();
14286        assert!(runner.prep().await.is_err());
14287        let saved = RunState::load(&runner.state.id).unwrap();
14288        assert!(saved.candidates.is_empty());
14289        let event = saved
14290            .events
14291            .iter()
14292            .find(|e| e.node == "prep-intent")
14293            .unwrap();
14294        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14295        assert_eq!(intent["branch"], branch);
14296        assert_eq!(intent["start"], runner.state.base_commit);
14297        assert_eq!(
14298            intent["worktree"],
14299            runner
14300                .state
14301                .worktree_root()
14302                .join(format!("cand-{label}"))
14303                .to_string_lossy()
14304                .as_ref()
14305        );
14306        assert_eq!(intent["picks"], serde_json::json!([]));
14307    }
14308
14309    #[tokio::test]
14310    async fn invalid_pinned_sources_create_no_partial_worktrees() {
14311        ask_test_home();
14312        let tmp = tempfile::tempdir().unwrap();
14313        let repo = tmp.path().join("repo");
14314        std::fs::create_dir_all(&repo).unwrap();
14315        init_repo(&repo);
14316        let mut runner = runner_at(RunStatus::Prep);
14317        runner.state.repo = repo.clone();
14318        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14319        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14320        runner.state.sources.pick_commits.push(refs::Input {
14321            token: "missing".into(),
14322            sha: "0".repeat(40),
14323            provenance: refs::Provenance::ExplicitCli,
14324        });
14325        assert!(runner.prep().await.is_err());
14326        assert!(runner.state.candidates.is_empty());
14327        assert!(!runner.state.worktree_root().exists());
14328    }
14329
14330    #[tokio::test]
14331    async fn prose_and_legacy_inference_never_import_into_candidates() {
14332        ask_test_home();
14333        let tmp = tempfile::tempdir().unwrap();
14334        let repo = tmp.path().join("repo");
14335        std::fs::create_dir_all(&repo).unwrap();
14336        init_repo(&repo);
14337        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14338        git::git(&repo, &["checkout", "-b", "magi/history/A"])
14339            .await
14340            .unwrap();
14341        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14342        git::git(&repo, &["add", "."]).await.unwrap();
14343        git::git(&repo, &["commit", "-m", "historical implementation"])
14344            .await
14345            .unwrap();
14346        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14347        git::git(&repo, &["checkout", "main"]).await.unwrap();
14348        for instruction in [
14349            format!("Example: magi/history/A at {source}"),
14350            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14351            format!("Never import magi/history/A or {source}"),
14352            format!("Conversation answer: I mentioned {source} only as background"),
14353        ] {
14354            let mut runner = runner_at(RunStatus::Prep);
14355            runner.state.repo = repo.clone();
14356            runner.state.base_commit = base.clone();
14357            runner.state.instruction = instruction.clone();
14358            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14359            runner.state.config.blind.commit_msg_hook = false;
14360            runner.state.seeds = vec![refs::Seed {
14361                token: "magi/history/A".into(),
14362                sha: source.clone(),
14363                kind: refs::SeedKind::Unmerged,
14364                branch: true,
14365                detail: String::new(),
14366            }];
14367            runner.roles.implementers = vec![spec("one")];
14368            assert_eq!(seeded_instruction(&runner.state), instruction);
14369            // Model an old record with inferred seeds but no structured input field.
14370            let mut legacy = serde_json::to_value(&runner.state).unwrap();
14371            legacy.as_object_mut().unwrap().remove("sources");
14372            runner.state = serde_json::from_value(legacy).unwrap();
14373            assert_eq!(runner.state.sources, refs::Sources::default());
14374            runner.state.save().unwrap();
14375            runner.state = RunState::load(&runner.state.id).unwrap();
14376            runner.prep().await.unwrap();
14377            for candidate in &runner.state.candidates {
14378                assert_eq!(
14379                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14380                    base
14381                );
14382                assert!(!candidate.worktree.join("unwanted").exists());
14383            }
14384        }
14385    }
14386
14387    #[tokio::test]
14388    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14389        ask_test_home();
14390        let tmp = tempfile::tempdir().unwrap();
14391        let repo = tmp.path().join("repo");
14392        std::fs::create_dir_all(&repo).unwrap();
14393        init_repo(&repo);
14394        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14395        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14396        std::fs::write(repo.join("referenced"), "required commit").unwrap();
14397        git::git(&repo, &["add", "."]).await.unwrap();
14398        git::git(&repo, &["commit", "-m", "required seed"])
14399            .await
14400            .unwrap();
14401        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14402        git::git(&repo, &["checkout", "main"]).await.unwrap();
14403        let mut runner = runner_at(RunStatus::Prep);
14404        runner.state.repo = repo.clone();
14405        runner.state.base_commit = base.clone();
14406        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14407        runner.state.config.graph.advise = false;
14408        runner.state.config.blind.commit_msg_hook = false;
14409        runner.state.sources =
14410            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14411                .await
14412                .unwrap();
14413        runner.roles.implementers = vec![spec("one"), spec("two")];
14414        runner.roles.judges = vec![spec("judge")];
14415        let label = blind::assign_labels(2, runner.state.seed)[0];
14416        let branch = runner.state.branch_for(label);
14417        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14418        git::worktree_add_branch(&repo, &worktree, &branch, &base)
14419            .await
14420            .unwrap();
14421        runner.state.candidates.push(Candidate {
14422            index: 0,
14423            label,
14424            agent: "one".into(),
14425            branch,
14426            worktree,
14427            summary: String::new(),
14428            stat: String::new(),
14429            files: 0,
14430            commits: 0,
14431            empty: false,
14432            failed: None,
14433            verified_noop: None,
14434            duration_ms: 0,
14435            folded: false,
14436        });
14437        runner.state.save().unwrap();
14438        // Reload the checkpoint immediately before its first cherry-pick.
14439        runner.state = RunState::load(&runner.state.id).unwrap();
14440        assert_eq!(runner.state.prep_complete, Some(false));
14441        runner.prep().await.unwrap();
14442        assert_eq!(runner.state.prep_complete, Some(true));
14443        assert_eq!(runner.state.candidates.len(), 2);
14444        for candidate in &runner.state.candidates {
14445            assert_eq!(
14446                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14447                git::tree_of(&repo, &seed).await.unwrap()
14448            );
14449            assert_eq!(
14450                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14451                "required commit"
14452            );
14453        }
14454        assert!(runner.state.worktree_root().join("judge-1").exists());
14455        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14456        runner.prep().await.unwrap();
14457        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14458
14459        runner.state.prep_complete = Some(false);
14460        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14461            run: Some(format!(
14462                "test -f '{}'",
14463                tmp.path().join("dependency-ready").display()
14464            )),
14465            ..Default::default()
14466        }];
14467        assert!(runner.prep().await.is_err());
14468        assert!(runner.state.prep_failure.is_some());
14469        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14470        runner.state = RunState::load(&runner.state.id).unwrap();
14471        assert!(
14472            runner.prep().await.is_err(),
14473            "correcting the cause alone must not auto-retry"
14474        );
14475        let failed_at = runner.state.events.last().unwrap().at;
14476        assert!(
14477            runner.retry_preparation(failed_at).is_err(),
14478            "old approvals cannot unlock the hold"
14479        );
14480        let mut task = crate::queue::Task::new(
14481            "retry".into(),
14482            "retry setup".into(),
14483            repo.clone(),
14484            crate::queue::Source::Human,
14485        );
14486        task.start(runner.state.id.clone());
14487        task.hold_machine(Some("setup failed".into()));
14488        task.release_operator("cli", None);
14489        runner
14490            .retry_preparation(task.resumption.as_ref().unwrap().at)
14491            .unwrap();
14492        runner.prep().await.unwrap();
14493        assert!(runner.state.prep_failure.is_none());
14494        assert_eq!(runner.state.prep_complete, Some(true));
14495        assert_eq!(runner.state.candidates.len(), 2);
14496        assert!(runner.state.worktree_root().join("judge-1").exists());
14497        for candidate in &runner.state.candidates {
14498            assert_eq!(
14499                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14500                git::tree_of(&repo, &seed).await.unwrap()
14501            );
14502        }
14503    }
14504
14505    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14506        let mut heads = Vec::new();
14507        for candidate in candidates {
14508            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14509        }
14510        heads
14511    }
14512
14513    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14514        ask_test_home();
14515        let tmp = tempfile::tempdir().unwrap();
14516        let repo = tmp.path().join("repo");
14517        std::fs::create_dir(&repo).unwrap();
14518        init_repo(&repo);
14519        let mut runner = runner_at(RunStatus::Reviewing);
14520        runner.state.repo = repo.clone();
14521        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14522        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14523        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14524            let branch = format!("candidate-{label}");
14525            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14526            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14527                .await
14528                .unwrap();
14529            if label == 'B' {
14530                std::fs::write(worktree.join("change"), "winner").unwrap();
14531                git::rescue_commit(&worktree, "candidate change")
14532                    .await
14533                    .unwrap();
14534            }
14535            runner.state.candidates.push(Candidate {
14536                index,
14537                label,
14538                agent: "alpha".into(),
14539                branch,
14540                worktree,
14541                summary: "completed reply".into(),
14542                stat: String::new(),
14543                files: usize::from(label == 'B'),
14544                commits: usize::from(label == 'B'),
14545                empty: label == 'A',
14546                failed: None,
14547                verified_noop: None,
14548                duration_ms: 1,
14549                folded: false,
14550            });
14551        }
14552        runner.state.judge_skipped = true;
14553        runner.state.tally = Some(Tally {
14554            first_choice: BTreeMap::new(),
14555            borda: BTreeMap::new(),
14556            winner: 'B',
14557            rankings: 0,
14558            unanimous_initial: false,
14559            deliberated: false,
14560            changed_votes: 0,
14561            unanimous_final: false,
14562            tie_break: None,
14563            judges: 0,
14564            present: 0,
14565            quorum: 0,
14566            met_quorum: true,
14567            uncontested: Some("only B produced a change".into()),
14568        });
14569        runner.state.save().unwrap();
14570        (tmp, runner)
14571    }
14572
14573    #[tokio::test]
14574    #[cfg(unix)]
14575    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14576        for (solo, failed_checkpoint) in
14577            [(true, false), (false, false), (true, true), (false, true)]
14578        {
14579            let (_tmp, mut runner) = preservation_resume_fixture().await;
14580            if solo {
14581                runner.state.candidates.retain(|c| c.label == 'A');
14582            }
14583            runner.state.candidates[0].empty = false;
14584            runner.state.candidates[0].summary.clear();
14585            runner.state.tally = None;
14586            runner.state.judge_skipped = false;
14587            runner.state.status = RunStatus::Implementing;
14588            runner.state.config.disk.min_free_bytes = u64::MAX;
14589            let mut implementer = spec("alpha");
14590            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14591            runner.roles.implementers = vec![implementer];
14592            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14593            if failed_checkpoint {
14594                std::fs::create_dir_all(&patch).unwrap();
14595            }
14596            runner.state.save().unwrap();
14597            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14598            runner.state = RunState::load(&runner.state.id).unwrap();
14599            assert_eq!(runner.state.status, RunStatus::Blocked);
14600            assert!(runner.state.interrupted_implementations.contains(&'A'));
14601            assert_eq!(runner.state.candidates[0].commits, 0);
14602            assert!(!runner.state.candidates[0].empty);
14603            assert!(runner.state.candidates[0].failed.is_none());
14604            if failed_checkpoint {
14605                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14606                std::fs::remove_dir(patch).unwrap();
14607            }
14608            runner.state.resource_block = None;
14609            runner.state.config.disk.min_free_bytes = 0;
14610            runner.roles.implementers[0].command = vec![
14611                "sh".into(),
14612                "-c".into(),
14613                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14614            ];
14615            runner.implement().await.unwrap();
14616            assert!(runner.state.interrupted_implementations.is_empty());
14617            assert!(runner.state.candidate_preservation.is_empty());
14618            assert_eq!(runner.state.candidates[0].commits, 1);
14619            assert!(!runner.state.candidates[0].empty);
14620            assert!(
14621                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14622                    .unwrap()
14623                    .contains("retried empty interrupted seat")
14624            );
14625            if !solo {
14626                assert_eq!(runner.state.candidates[1].commits, 1);
14627            }
14628        }
14629    }
14630
14631    #[tokio::test]
14632    #[cfg(unix)]
14633    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14634        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14635            let (_tmp, mut runner) = preservation_resume_fixture().await;
14636            runner.state.candidates.retain(|c| c.label == 'A');
14637            runner.state.candidates[0].empty = false;
14638            runner.state.candidates[0].summary.clear();
14639            runner.state.tally = None;
14640            runner.state.judge_skipped = false;
14641            runner.state.status = RunStatus::Implementing;
14642            // Force the real post-spawn watchdog, independently of available
14643            // disk space. Dirty/committed work predates its first measurement.
14644            runner.state.config.disk.min_free_bytes = u64::MAX;
14645            let wt = runner.state.candidates[0].worktree.clone();
14646            std::fs::write(
14647                wt.join("interrupted-source"),
14648                "retain implementation work\n",
14649            )
14650            .unwrap();
14651            if committed {
14652                git::rescue_commit(&wt, "implementation checkpoint")
14653                    .await
14654                    .unwrap();
14655            }
14656            let lock = git::git(
14657                &wt,
14658                &[
14659                    "rev-parse",
14660                    "--path-format=absolute",
14661                    "--git-path",
14662                    "index.lock",
14663                ],
14664            )
14665            .await
14666            .unwrap();
14667            if block_rescue {
14668                std::fs::write(&lock, "fixture-owned lock").unwrap();
14669            }
14670            let mut implementer = spec("alpha");
14671            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14672            runner.roles.implementers = vec![implementer];
14673            runner.state.save().unwrap();
14674            let result = runner.implement().await;
14675            assert_eq!(result.is_err(), block_rescue);
14676            assert_eq!(
14677                runner.state.resource_block.as_ref().unwrap().kind,
14678                "disk pressure"
14679            );
14680            assert!(
14681                runner
14682                    .state
14683                    .events
14684                    .iter()
14685                    .any(|e| e.message.contains("agent execution interrupted"))
14686            );
14687            assert!(
14688                runner
14689                    .state
14690                    .dir()
14691                    .join("artifacts/candidate-A-recovery/staged.patch")
14692                    .exists()
14693            );
14694            if block_rescue {
14695                let saved = RunState::load(&runner.state.id).unwrap();
14696                assert!(saved.candidate_preservation.contains_key(&'A'));
14697                assert!(!saved.candidates[0].empty);
14698                assert_eq!(
14699                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14700                    "retain implementation work\n"
14701                );
14702                std::fs::remove_file(lock).unwrap();
14703                runner.state = saved;
14704            } else {
14705                assert_eq!(runner.state.status, RunStatus::Blocked);
14706                assert_eq!(runner.state.candidates[0].commits, 1);
14707            }
14708            // Recovery must save the candidate before deciding whether another
14709            // implementation is needed. No implementer is available on retry.
14710            runner.roles.implementers.clear();
14711            runner.state.config.disk.min_free_bytes = 0;
14712            runner.state.resource_block = None;
14713            runner.implement().await.unwrap();
14714            assert!(runner.state.candidate_preservation.is_empty());
14715            assert_eq!(runner.state.candidates[0].commits, 1);
14716            assert!(!runner.state.candidates[0].empty);
14717            assert!(git::is_clean(&wt).await.unwrap());
14718            assert!(
14719                crate::run::read_artifact(&runner.state, "cand-A.patch")
14720                    .unwrap()
14721                    .contains("retain implementation work")
14722            );
14723        }
14724    }
14725
14726    #[tokio::test]
14727    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14728        let (_tmp, mut runner) = preservation_resume_fixture().await;
14729        // Schema 15 already confirmed the unfolded no-change result.
14730        runner.implement().await.unwrap();
14731        assert!(runner.state.candidate_preservation.is_empty());
14732        assert!(
14733            !runner
14734                .state
14735                .dir()
14736                .join("artifacts/candidate-A-recovery")
14737                .exists()
14738        );
14739        runner.fold_losers().await.unwrap();
14740        assert!(!runner.state.candidates[0].worktree.exists());
14741        for schema in [14, crate::run::SCHEMA] {
14742            runner.state.schema = schema;
14743            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14744            runner.state.save().unwrap();
14745            runner.state = RunState::load(&runner.state.id).unwrap();
14746            runner.implement().await.unwrap();
14747            assert!(runner.state.candidate_preservation.is_empty());
14748            assert!(runner.state.candidates[0].folded);
14749            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14750        }
14751    }
14752
14753    #[tokio::test]
14754    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14755        let (_tmp, mut runner) = preservation_resume_fixture().await;
14756        runner.state.candidates.retain(|c| c.label == 'A');
14757        runner.state.tally = None;
14758        runner.state.judge_skipped = false;
14759        let candidate = &mut runner.state.candidates[0];
14760        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14761        candidate.failed = Some("deployment validation unavailable".into());
14762        std::fs::write(
14763            candidate.worktree.join("repair"),
14764            "future launcher policy\n",
14765        )
14766        .unwrap();
14767        runner
14768            .state
14769            .candidate_preservation
14770            .insert('A', "awaiting persistence".into());
14771        runner.persist_candidates().await.unwrap();
14772        let candidate = &runner.state.candidates[0];
14773        assert!(!candidate.empty);
14774        assert_eq!(candidate.commits, 1);
14775        assert_eq!(candidate.files, 1);
14776        assert!(candidate.failed.is_none());
14777        assert!(candidate.summary.contains("EPERM"));
14778        assert!(
14779            crate::run::read_artifact(&runner.state, "cand-A.patch")
14780                .unwrap()
14781                .contains("future launcher policy")
14782        );
14783        assert!(runner.state.candidate_preservation.is_empty());
14784    }
14785
14786    #[tokio::test]
14787    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14788        let (_tmp, mut runner) = preservation_resume_fixture().await;
14789        let wt = runner.state.candidates[0].worktree.clone();
14790        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14791        runner.state.schema = 14;
14792        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14793        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14794        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14795        for _ in 0..2 {
14796            assert!(runner.implement().await.is_err());
14797            assert_eq!(runner.state.status, RunStatus::Blocked);
14798            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14799            assert!(
14800                runner.state.candidates[0].empty,
14801                "old verdict must remain unusable"
14802            );
14803            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14804            assert!(runner.state.judge_skipped);
14805            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14806            assert!(
14807                crate::run::read_artifact(&runner.state, "cand-A.patch")
14808                    .unwrap()
14809                    .contains("owned edits")
14810            );
14811            assert!(wt.exists());
14812            assert_eq!(
14813                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14814                    .await
14815                    .unwrap(),
14816                1
14817            );
14818            runner.state = RunState::load(&runner.state.id).unwrap();
14819        }
14820    }
14821
14822    #[tokio::test]
14823    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14824        let (_tmp, mut runner) = preservation_resume_fixture().await;
14825        let wt = runner.state.candidates[0].worktree.clone();
14826        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14827        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14828        let index_lock = PathBuf::from(
14829            git::git(
14830                &wt,
14831                &[
14832                    "rev-parse",
14833                    "--path-format=absolute",
14834                    "--git-path",
14835                    "index.lock",
14836                ],
14837            )
14838            .await
14839            .unwrap(),
14840        );
14841        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14842        runner.state.schema = 14;
14843        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14844        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14845        assert!(runner.implement().await.is_err());
14846        assert!(runner.after_implement().is_err());
14847        assert!(runner.fold_losers().await.is_err());
14848        assert_eq!(
14849            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14850            "failed seat edits"
14851        );
14852        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14853        runner.state = RunState::load(&runner.state.id).unwrap();
14854        std::fs::remove_file(index_lock).unwrap();
14855        // Recovery finds changes that the old uncontested result never judged.
14856        assert!(runner.implement().await.is_err());
14857        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14858        assert!(runner.state.candidates[0].empty);
14859        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14860        assert!(
14861            crate::run::read_artifact(&runner.state, "cand-A.patch")
14862                .unwrap()
14863                .contains("failed seat edits")
14864        );
14865        assert!(wt.exists());
14866    }
14867
14868    #[tokio::test]
14869    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14870        let (_tmp, mut runner) = preservation_resume_fixture().await;
14871        let repo = runner.state.repo.clone();
14872        let common = git::git(
14873            &repo,
14874            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14875        )
14876        .await
14877        .unwrap();
14878        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14879        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14880        assert!(runner.fold_losers().await.is_err());
14881        assert!(!runner.state.candidates[0].worktree.exists());
14882        assert!(!runner.state.candidates[0].folded);
14883        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14884        runner.state = RunState::load(&runner.state.id).unwrap();
14885        std::fs::remove_file(branch_lock).unwrap();
14886        let judge = runner.state.worktree_root().join("judge-1");
14887        git::worktree_add_detached(&repo, &judge, "main")
14888            .await
14889            .unwrap();
14890        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14891        runner.roles.judges = vec![spec("judge")];
14892        assert!(runner.fold_losers().await.is_err());
14893        runner.state = RunState::load(&runner.state.id).unwrap();
14894        assert!(
14895            runner.state.candidates[0].folded,
14896            "candidate progress saved before judge failure"
14897        );
14898        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14899        assert!(judge.join("uncommitted").exists());
14900        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14901        runner.fold_losers().await.unwrap();
14902        assert!(!judge.exists());
14903        runner.fold_losers().await.unwrap();
14904    }
14905
14906    #[tokio::test]
14907    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14908        let (_tmp, mut runner) = preservation_resume_fixture().await;
14909        runner.state.candidates.retain(|c| c.label == 'B');
14910        runner.state.tally = None;
14911        runner.state.judge_skipped = false;
14912        runner.state.prep_complete = Some(true);
14913        runner.state.config.graph.advise = false;
14914        runner.state.config.graph.selected_implementer = Some("alpha".into());
14915        let wt = runner.state.candidates[0].worktree.clone();
14916        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14917        runner
14918            .state
14919            .candidate_preservation
14920            .insert('B', "awaiting persistence".into());
14921        let index_lock = git::git(
14922            &wt,
14923            &[
14924                "rev-parse",
14925                "--path-format=absolute",
14926                "--git-path",
14927                "index.lock",
14928            ],
14929        )
14930        .await
14931        .unwrap();
14932        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14933        runner.state.save().unwrap();
14934        assert!(runner.execute_graph().await.is_err());
14935        assert_eq!(runner.state.status, RunStatus::Blocked);
14936        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14937        std::fs::remove_file(index_lock).unwrap();
14938        // Park at the next graph boundary: this tests the real selected-seat
14939        // hold guard without dispatching reviewers or publishing anything.
14940        for _ in 0..2 {
14941            runner.state = RunState::load(&runner.state.id).unwrap();
14942            runner.pause.park();
14943            runner.execute_graph().await.unwrap();
14944            assert_eq!(runner.state.status, RunStatus::Implementing);
14945            assert!(runner.state.candidate_preservation.is_empty());
14946            assert_eq!(runner.state.candidates[0].commits, 2);
14947            assert!(runner.state.tally.is_none());
14948        }
14949        runner.implement().await.unwrap();
14950        assert_eq!(runner.state.status, RunStatus::Judging);
14951        // A genuine selected-implementer hold must still require release.
14952        runner.state.status = RunStatus::Blocked;
14953        runner.state.candidates[0].failed = Some("implementation failed".into());
14954        runner.state.save().unwrap();
14955        let error = runner.execute_graph().await.unwrap_err();
14956        assert!(
14957            error
14958                .to_string()
14959                .contains("selected implementation is held")
14960        );
14961    }
14962
14963    #[tokio::test]
14964    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14965        let (_tmp, mut runner) = preservation_resume_fixture().await;
14966        let repo = runner.state.repo.clone();
14967        let candidate = runner.state.candidates[0].clone();
14968        // Remove only the disposable fixture directory, leaving Git's linked
14969        // worktree registration and branch intact, as in a vanished checkout.
14970        let registered = git::worktree_holding(&repo, &candidate.branch)
14971            .await
14972            .unwrap()
14973            .unwrap();
14974        assert_eq!(
14975            registered.canonicalize().unwrap(),
14976            candidate.worktree.canonicalize().unwrap()
14977        );
14978        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14979        assert_eq!(
14980            git::worktree_holding(&repo, &candidate.branch)
14981                .await
14982                .unwrap(),
14983            Some(registered)
14984        );
14985        assert!(
14986            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14987            "the stale registration initially keeps the branch checked out"
14988        );
14989        runner.fold_losers().await.unwrap();
14990        assert!(runner.state.candidates[0].folded);
14991        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14992        assert_eq!(
14993            git::worktree_holding(&repo, &candidate.branch)
14994                .await
14995                .unwrap(),
14996            None
14997        );
14998        runner.state = RunState::load(&runner.state.id).unwrap();
14999        runner.fold_losers().await.unwrap();
15000    }
15001
15002    #[tokio::test]
15003    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15004        let (_tmp, mut runner) = preservation_resume_fixture().await;
15005        let repo = runner.state.repo.clone();
15006        let candidate = runner.state.candidates[0].clone();
15007        let winner = runner.state.candidates[1].clone();
15008        let registered = git::worktree_holding(&repo, &candidate.branch)
15009            .await
15010            .unwrap()
15011            .unwrap();
15012        assert_eq!(
15013            registered.canonicalize().unwrap(),
15014            candidate.worktree.canonicalize().unwrap()
15015        );
15016        std::fs::remove_dir_all(&candidate.worktree).unwrap();
15017        assert_eq!(
15018            git::worktree_holding(&repo, &candidate.branch)
15019                .await
15020                .unwrap(),
15021            Some(registered)
15022        );
15023        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15024
15025        for _ in 0..2 {
15026            fold_run(&mut runner.state, false, &crate::run::home())
15027                .await
15028                .unwrap();
15029            runner.state = RunState::load(&runner.state.id).unwrap();
15030            assert!(runner.state.candidates[0].folded);
15031            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15032            assert_eq!(
15033                git::worktree_holding(&repo, &candidate.branch)
15034                    .await
15035                    .unwrap(),
15036                None
15037            );
15038            assert!(!runner.state.candidates[1].folded);
15039            assert!(winner.worktree.exists());
15040            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15041        }
15042    }
15043
15044    #[tokio::test]
15045    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15046        ask_test_home();
15047        let tmp = tempfile::tempdir().unwrap();
15048        let repo = tmp.path().join("repo");
15049        std::fs::create_dir(&repo).unwrap();
15050        init_repo(&repo);
15051        let mut runner = runner_at(RunStatus::Implementing);
15052        runner.state.repo = repo.clone();
15053        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15054        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15055        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15056            let branch = format!("candidate-{label}");
15057            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15058            git::worktree_add_branch(&repo, &wt, &branch, "main")
15059                .await
15060                .unwrap();
15061            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15062            git::git(&wt, &["add", "README.md"]).await.unwrap();
15063            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15064            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15065            runner.state.candidates.push(Candidate {
15066                index,
15067                label,
15068                agent: "alpha".to_owned(),
15069                branch,
15070                worktree: wt,
15071                summary: "completed implementation".to_owned(),
15072                stat: String::new(),
15073                files: 0,
15074                commits: 0,
15075                empty: false,
15076                failed: None,
15077                verified_noop: None,
15078                duration_ms: 42,
15079                folded: false,
15080            });
15081            runner
15082                .state
15083                .candidate_preservation
15084                .insert(label, "awaiting persistence".to_owned());
15085        }
15086        let wt = runner.state.candidates[0].worktree.clone();
15087        let index_lock = git::git(
15088            &wt,
15089            &[
15090                "rev-parse",
15091                "--path-format=absolute",
15092                "--git-path",
15093                "index.lock",
15094            ],
15095        )
15096        .await
15097        .unwrap();
15098        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15099        runner.state.save().unwrap();
15100        assert!(runner.persist_candidates().await.is_err());
15101        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15102        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15103        assert_eq!(
15104            runner.state.candidates[1].commits, 1,
15105            "other results are still collected"
15106        );
15107        assert!(
15108            !runner.state.candidates[0].empty,
15109            "failed rescue is not no change"
15110        );
15111        assert_eq!(
15112            runner.state.candidates[0].summary,
15113            "completed implementation"
15114        );
15115        assert!(runner.after_implement().is_err());
15116        let home = tmp.path().join("fold-home");
15117        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15118        assert!(wt.join("new-file").exists());
15119        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15120        assert!(recovery.join("staged.patch").exists());
15121        assert!(recovery.join("unstaged.patch").exists());
15122        assert_eq!(
15123            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15124            "untracked edits\n"
15125        );
15126        runner.state.begin_pass();
15127        runner.state.end_pass_parked(ParkCause::Upgrade {
15128            from: "1.0".into(),
15129            to: Some("1.1".into()),
15130        });
15131        runner.state.save().unwrap();
15132        runner.state = RunState::load(&runner.state.id).unwrap();
15133        assert!(matches!(
15134            runner.state.passes.last().unwrap().end,
15135            Some(crate::run::PassEnd::Parked {
15136                cause: ParkCause::Upgrade { .. }
15137            })
15138        ));
15139        std::fs::remove_file(index_lock).unwrap();
15140        // No implementer is configured: a persistence retry cannot dispatch a seat.
15141        runner.implement().await.unwrap();
15142        assert!(runner.state.candidate_preservation.is_empty());
15143        assert!(
15144            runner
15145                .state
15146                .candidates
15147                .iter()
15148                .all(|c| c.commits == 1 && !c.empty)
15149        );
15150        assert!(git::is_clean(&wt).await.unwrap());
15151        assert!(
15152            crate::run::read_artifact(&runner.state, "cand-A.patch")
15153                .unwrap()
15154                .contains("later tracked edits")
15155        );
15156    }
15157
15158    #[tokio::test]
15159    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15160        ask_test_home();
15161        let tmp = tempfile::tempdir().unwrap();
15162        let repo = tmp.path().join("repo");
15163        std::fs::create_dir(&repo).unwrap();
15164        init_repo(&repo);
15165        let wt = tmp.path().join("candidate");
15166        git::worktree_add_branch(&repo, &wt, "candidate", "main")
15167            .await
15168            .unwrap();
15169        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15170        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15171        assert!(wt.join("uncommitted").exists());
15172        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15173        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15174        let mut runner = runner_at(RunStatus::Implementing);
15175        runner.state.repo = repo.clone();
15176        runner.state.base_commit = "missing-base".to_owned();
15177        runner.state.candidates.push(Candidate {
15178            index: 0,
15179            label: 'A',
15180            agent: "alpha".into(),
15181            branch: "candidate".into(),
15182            worktree: wt.clone(),
15183            summary: "completed".into(),
15184            stat: String::new(),
15185            files: 0,
15186            commits: 0,
15187            empty: false,
15188            failed: None,
15189            verified_noop: None,
15190            duration_ms: 1,
15191            folded: false,
15192        });
15193        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15194        assert!(
15195            fold_run(&mut runner.state, true, &tmp.path().join("home"))
15196                .await
15197                .is_err()
15198        );
15199        assert!(!runner.state.candidates[0].folded);
15200        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15201        std::fs::remove_file(wt.join("uncommitted")).unwrap();
15202        runner
15203            .state
15204            .candidate_preservation
15205            .insert('A', "pending".into());
15206        assert!(runner.persist_candidates().await.is_err());
15207        assert!(!runner.state.candidates[0].empty);
15208        assert!(!runner.state.candidates[0].folded);
15209        assert!(wt.exists());
15210        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15211        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15212        std::fs::create_dir_all(&patch_path).unwrap();
15213        assert!(
15214            runner.persist_candidates().await.is_err(),
15215            "artifact failure stays pending"
15216        );
15217        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15218        assert!(!runner.state.candidates[0].empty);
15219        std::fs::remove_dir(patch_path).unwrap();
15220        runner.persist_candidates().await.unwrap();
15221        assert!(
15222            runner.state.candidates[0].empty,
15223            "only successful inspection confirms no change"
15224        );
15225    }
15226
15227    fn init_repo(dir: &Path) {
15228        let run = |args: &[&str]| {
15229            let out = std::process::Command::new("git")
15230                .args(args)
15231                .current_dir(dir)
15232                .quiet()
15233                .output()
15234                .expect("spawn git");
15235            assert!(
15236                out.status.success(),
15237                "git {args:?} failed: {}",
15238                String::from_utf8_lossy(&out.stderr)
15239            );
15240        };
15241        run(&["init", "-b", "main"]);
15242        run(&["config", "user.name", "magi test"]);
15243        run(&["config", "user.email", "magi@example.com"]);
15244        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15245        run(&["add", "-A"]);
15246        run(&["commit", "-m", "init"]);
15247    }
15248
15249    // `settle_questions` is what closes the ghost the phone showed: a run's
15250    // seat asked something, the run then ended, and nothing was left to
15251    // abandon the question it left `open`. `HOME` is a process-wide
15252    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
15253    // first time it runs in the binary — every test below still reaches the
15254    // same directory whichever call won, and each gets its own run id from
15255    // `RunState::new`, so they never collide there.
15256    fn ask_test_home() {
15257        crate::run::pin_test_home();
15258    }
15259
15260    #[cfg(unix)]
15261    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15262        ask_test_home();
15263        let tmp = tempfile::tempdir().unwrap();
15264        let repo = tmp.path().join("repo");
15265        std::fs::create_dir(&repo).unwrap();
15266        init_repo(&repo);
15267        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15268        let origin = tmp.path().join("origin.git");
15269        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15270            .await
15271            .unwrap();
15272        git::git(
15273            &repo,
15274            &["remote", "add", "origin", origin.to_str().unwrap()],
15275        )
15276        .await
15277        .unwrap();
15278        git::git(&repo, &["checkout", "-b", "fix/retained"])
15279            .await
15280            .unwrap();
15281        std::fs::write(repo.join("retained"), "retained change").unwrap();
15282        git::git(&repo, &["add", "."]).await.unwrap();
15283        git::git(&repo, &["commit", "-m", "retained implementation"])
15284            .await
15285            .unwrap();
15286        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15287        git::git(&repo, &["checkout", "main"]).await.unwrap();
15288        std::fs::write(repo.join("base-change"), "new base").unwrap();
15289        git::git(&repo, &["add", "."]).await.unwrap();
15290        git::git(&repo, &["commit", "-m", "base moved"])
15291            .await
15292            .unwrap();
15293        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15294        git::git(&repo, &["checkout", "fix/retained"])
15295            .await
15296            .unwrap();
15297        let mut runner = runner_at(RunStatus::Voting);
15298        runner.state.repo = repo.clone();
15299        runner.state.base_commit = base;
15300        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15301        runner.state.config.disk.min_free_bytes = 0;
15302        runner.state.config.graph.review_rounds = 1;
15303        runner.state.config.verify.e2e = vec!["true".into()];
15304        runner.state.config.verify.gate = vec!["true".into()];
15305        runner.state.candidates = vec![Candidate {
15306            index: 0,
15307            label: 'A',
15308            agent: "fixture".into(),
15309            branch: "fix/retained".into(),
15310            worktree: repo.clone(),
15311            summary: String::new(),
15312            stat: String::new(),
15313            files: 1,
15314            commits: 1,
15315            empty: false,
15316            failed: None,
15317            verified_noop: None,
15318            duration_ms: 0,
15319            folded: false,
15320        }];
15321        runner.state.tally = Some(Tally {
15322            first_choice: BTreeMap::new(),
15323            borda: BTreeMap::new(),
15324            winner: 'A',
15325            rankings: 0,
15326            unanimous_initial: true,
15327            deliberated: false,
15328            changed_votes: 0,
15329            unanimous_final: true,
15330            tie_break: None,
15331            judges: 0,
15332            present: 0,
15333            quorum: 0,
15334            met_quorum: true,
15335            uncontested: Some("review-only".into()),
15336        });
15337        let mut task = crate::queue::Task::new(
15338            "fixture".into(),
15339            "fixture".into(),
15340            repo,
15341            crate::queue::Source::Human,
15342        );
15343        task.link_run(&runner.state.id);
15344        crate::queue::Queue::open().put(&mut task).unwrap();
15345        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15346        runner.state.base_sync = Some(BaseSync {
15347            tip: String::new(),
15348            behind: 1,
15349            attempts: BASE_SYNC_ROUNDS,
15350            conflict: None,
15351            already_in: None,
15352        });
15353        runner.sync_to_base().await.unwrap();
15354        assert_eq!(runner.state.status, RunStatus::Blocked);
15355        (tmp, runner, head, task.id)
15356    }
15357
15358    #[cfg(unix)]
15359    #[tokio::test]
15360    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15361        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15362        if std::env::var_os(CHILD).is_none() {
15363            let home = tempfile::tempdir().unwrap();
15364            let output = std::process::Command::new(std::env::current_exe().unwrap())
15365                .args([
15366                    "--exact",
15367                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15368                    "--nocapture",
15369                ])
15370                .env(CHILD, "1")
15371                .env("MAGI_HOME", home.path())
15372                .env("MAGI_WORKER_HOME", home.path())
15373                .env("MAGI_WORKER_ID", "abcdef")
15374                .output()
15375                .unwrap();
15376            assert!(
15377                output.status.success()
15378                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15379                "{}\n{}",
15380                String::from_utf8_lossy(&output.stdout),
15381                String::from_utf8_lossy(&output.stderr)
15382            );
15383            return;
15384        }
15385        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15386        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15387        let home = crate::run::home();
15388        let queue = crate::queue::Queue::open();
15389        let mut owner = queue.get(&task).unwrap();
15390        owner.release_operator("test-release", None);
15391        owner.start(runner.state.id.clone());
15392        queue.put(&mut owner).unwrap();
15393        runner.state.driver_exited = true;
15394        runner.state.task_instruction = Some(owner.instruction.clone());
15395        runner.state.save().unwrap();
15396        let _claim = queue.claim(&task).unwrap();
15397        let reserve = |id: &str| {
15398            let directory = home.join("workers/jobs").join(id);
15399            std::fs::create_dir_all(&directory).unwrap();
15400            let job = crate::worker::Job {
15401                id: id.into(),
15402                sequence: 1,
15403                kind: "task".into(),
15404                key: format!("task:{task}"),
15405                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15406                generation: "fixture-generation".into(),
15407                version: None,
15408                status: crate::worker::Status::Running,
15409                result: None,
15410                pid: Some(std::process::id()),
15411                created_at: 0,
15412                finished_at: None,
15413            };
15414            std::fs::write(
15415                directory.join("job.json"),
15416                serde_json::to_vec(&job).unwrap(),
15417            )
15418            .unwrap();
15419            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15420        };
15421        let _own = reserve("abcdef");
15422        let other = reserve("fedcba");
15423        assert!(
15424            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15425                .await
15426                .is_err()
15427        );
15428        assert!(
15429            RunState::load(&runner.state.id)
15430                .unwrap()
15431                .base_sync_recovery
15432                .authorizations
15433                .is_empty()
15434        );
15435        drop(other);
15436        assert!(crate::daemon::is_working_on(
15437            &home,
15438            &runner.state.id,
15439            Timestamp::now()
15440        ));
15441        assert!(
15442            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15443        );
15444        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15445            .await
15446            .unwrap();
15447        assert!(
15448            RunState::load(&runner.state.id)
15449                .unwrap()
15450                .base_recovery_pending()
15451        );
15452    }
15453
15454    #[cfg(unix)]
15455    #[tokio::test]
15456    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15457        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15458        let home = crate::run::home();
15459        runner.state.driver_pid = Some(std::process::id());
15460        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15461        runner.state.driver_exited = false;
15462        runner.state.begin_pass();
15463        runner.state.save().unwrap();
15464        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15465        driver.record(&runner.state).unwrap();
15466        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15467        // Preserve the durable claim as if the process died before Drop saved.
15468        drop(driver);
15469        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15470        assert_eq!(
15471            runner.state.liveness_under(&home, false),
15472            crate::run::Liveness::Dead
15473        );
15474        let queue = crate::queue::Queue::open();
15475        let _claim = queue.claim(&task).unwrap();
15476        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15477            .await
15478            .unwrap();
15479        let disk = RunState::load(&runner.state.id).unwrap();
15480        assert!(!disk.driver_exited);
15481        assert!(disk.base_recovery_pending());
15482    }
15483
15484    #[cfg(unix)]
15485    #[tokio::test]
15486    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15487        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15488        if std::env::var_os(CHILD).is_none() {
15489            use std::os::unix::fs::PermissionsExt;
15490            let tmp = tempfile::tempdir().unwrap();
15491            let gh = tmp.path().join("gh");
15492            std::fs::write(&gh, r#"#!/bin/sh
15493case "$1 $2" in
15494  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15495  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15496  *) exit 2 ;;
15497esac
15498"#).unwrap();
15499            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15500            let mut paths = vec![tmp.path().to_path_buf()];
15501            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15502            let output = std::process::Command::new(std::env::current_exe().unwrap())
15503                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15504                .env(CHILD, "1")
15505                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15506                .env("PATH", std::env::join_paths(paths).unwrap())
15507                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15508                .output().unwrap();
15509            assert!(
15510                output.status.success()
15511                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15512                "{}\n{}",
15513                String::from_utf8_lossy(&output.stdout),
15514                String::from_utf8_lossy(&output.stderr)
15515            );
15516            return;
15517        }
15518        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15519        let repo = runner.state.repo.clone();
15520        git::git(&repo, &["checkout", "main"]).await.unwrap();
15521        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15522        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15523        git::git(&repo, &["checkout", "fix/retained"])
15524            .await
15525            .unwrap();
15526        runner.state.pr = Some(crate::run::PrRecord {
15527            url: "https://example.test/project/pull/1".into(),
15528            number: 1,
15529            state: "open".into(),
15530            checks: "unknown".into(),
15531            round: 0,
15532            rounds: 1,
15533            red_at_merge: Vec::new(),
15534        });
15535        runner.sync_to_base().await.unwrap();
15536        assert_eq!(runner.state.status, RunStatus::Blocked);
15537        assert_eq!(
15538            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15539            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15540        );
15541        assert!(runner.state.base_sync_closure_pending());
15542        assert!(crate::handover::can_resume(&runner.state));
15543        let queue = crate::queue::Queue::open();
15544        assert!(!crate::base_sync_recovery::held_task(
15545            &queue,
15546            &queue.get(&task).unwrap()
15547        ));
15548        assert!(runner.state.authorize_base_sync(&task).is_err());
15549        // Legacy closure stops have no typed stop but must retain the checkpoint.
15550        runner.state.base_sync_recovery.stop = None;
15551        runner.state.status = RunStatus::Voting;
15552        runner.state.gate_ran = true;
15553        runner.state.config.graph.review_rounds = 0;
15554        runner.state.save().unwrap();
15555        assert!(crate::handover::can_resume(&runner.state));
15556        runner.state = RunState::load(&runner.state.id).unwrap();
15557        runner.execute_graph().await.unwrap();
15558        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15559        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15560        assert_eq!(
15561            runner.state.base_sync.as_ref().unwrap().attempts,
15562            BASE_SYNC_ROUNDS
15563        );
15564        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15565        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15566        assert!(runner.state.reviews.is_empty());
15567        assert!(runner.state.gate.is_empty());
15568    }
15569
15570    #[cfg(unix)]
15571    #[tokio::test]
15572    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15573        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15574        runner.state.driver_exited = true;
15575        runner.state.save().unwrap();
15576        let home = crate::run::home();
15577        let queue = crate::queue::Queue::open();
15578        let _claim = queue.claim(&task).unwrap();
15579        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15580        assert!(
15581            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15582                .await
15583                .is_err()
15584        );
15585        drop(lease);
15586        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15587        assert!(
15588            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15589                .await
15590                .is_err()
15591        );
15592        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15593        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15594            .await
15595            .unwrap();
15596        let disk = RunState::load(&runner.state.id).unwrap();
15597        assert!(disk.base_recovery_pending());
15598        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15599        assert!(
15600            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15601                .await
15602                .is_err()
15603        );
15604        assert_eq!(
15605            RunState::load(&runner.state.id)
15606                .unwrap()
15607                .base_sync_recovery
15608                .authorizations
15609                .len(),
15610            1
15611        );
15612        let mut owner = queue.get(&task).unwrap();
15613        owner.runs.push("20260101-000000-ffff".into());
15614        queue.put(&mut owner).unwrap();
15615        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15616            .await
15617            .unwrap_err();
15618        assert!(error.to_string().contains("run task ownership changed"));
15619    }
15620
15621    #[cfg(unix)]
15622    #[tokio::test]
15623    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15624        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15625        let mut legacy = serde_json::to_value(&runner.state).unwrap();
15626        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15627        legacy["status"] = serde_json::json!("voting");
15628        runner.state = serde_json::from_value(legacy).unwrap();
15629        runner.state.save().unwrap();
15630        for _ in 0..3 {
15631            runner.execute_graph().await.unwrap();
15632            assert_eq!(runner.state.status, RunStatus::Blocked);
15633            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15634            assert!(!crate::handover::can_resume(&runner.state));
15635            assert_eq!(
15636                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15637                head
15638            );
15639            assert!(runner.state.reviews.is_empty());
15640            assert!(!runner.state.gate_ran);
15641        }
15642        let notice = crate::notices::run_ended(&runner.state).unwrap();
15643        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15644        runner.state.begin_pass();
15645        runner.state.status = RunStatus::Voting;
15646        runner.deliberate().await.unwrap();
15647        runner.vote().await.unwrap();
15648        runner.finish_after_tally().await.unwrap();
15649        assert_eq!(runner.state.status, RunStatus::Blocked);
15650        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15651        assert_eq!(
15652            crate::notices::run_ended(&runner.state).unwrap().since,
15653            notice.since
15654        );
15655    }
15656
15657    #[cfg(unix)]
15658    #[tokio::test]
15659    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15660        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15661        for detail in [
15662            "fix/retained conflicts with origin/main and did not rebase: conflict",
15663            "rebased fix/retained locally but origin refused the push",
15664            "unknown failure",
15665        ] {
15666            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15667            runner.state.record_base_sync_stop(&head);
15668            assert!(runner.state.authorize_base_sync(&task).is_err());
15669            runner.state.status = RunStatus::Voting;
15670            runner.execute_graph().await.unwrap();
15671            assert_eq!(runner.state.status, RunStatus::Blocked);
15672            assert!(!crate::handover::can_resume(&runner.state));
15673            assert_eq!(
15674                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15675                head
15676            );
15677        }
15678    }
15679
15680    #[cfg(unix)]
15681    #[tokio::test]
15682    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15683        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15684        runner.state.authorize_base_sync(&task).unwrap();
15685        assert!(runner.state.consume_base_recovery(&head));
15686        runner.state.save().unwrap();
15687        runner.state = RunState::load(&runner.state.id).unwrap();
15688        runner.execute_graph().await.unwrap();
15689        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15690        assert_eq!(runner.state.status, RunStatus::Blocked);
15691        assert!(!runner.state.consume_base_recovery(&head));
15692        assert!(!crate::handover::can_resume(&runner.state));
15693        runner.state.authorize_base_sync(&task).unwrap();
15694        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15695        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15696        assert_eq!(
15697            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15698            head
15699        );
15700    }
15701
15702    #[cfg(unix)]
15703    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
15704        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
15705        // This is an ordinary gate stop, not an authorized base-budget recovery.
15706        runner.state.prep_complete = Some(true);
15707        runner.state.base_sync = None;
15708        runner.state.base_sync_recovery = Default::default();
15709        runner.state.config.graph.gate_fix_rounds = 1;
15710        let mut review = review_round(true, 0, 1, 1, false, true);
15711        review.head = head.clone();
15712        review.verified_head = Some(head.clone());
15713        runner.state.reviews = vec![review];
15714        runner.state.gate = vec![CommandOutcome {
15715            command: "old gate".into(),
15716            code: Some(1),
15717            output_tail: "old failure".into(),
15718            duration_ms: 0,
15719            resource_blocked: false,
15720        }];
15721        runner.state.gate_head = Some(head.clone());
15722        runner.state.gate_ran = true;
15723        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
15724            agent: "fixture".into(),
15725            failed: runner.state.gate.clone(),
15726            notes: "spent".into(),
15727            committed: false,
15728            error: None,
15729        }];
15730        runner.state.land_approval = Some(crate::run::LandApproval {
15731            question: "old approval".into(),
15732            head: head.clone(),
15733        });
15734        runner.state.status = RunStatus::Blocked;
15735        (tmp, runner, head)
15736    }
15737
15738    #[cfg(unix)]
15739    #[tokio::test]
15740    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
15741        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
15742            let (tmp, mut runner, old) = resumed_gate_fixture().await;
15743            let phases = tmp.path().join("phases");
15744            runner.state.config.verify.e2e = vec![format!(
15745                "echo e2e >> '{}'; {}",
15746                phases.display(),
15747                if e2e_ok {
15748                    "true"
15749                } else {
15750                    "echo new-e2e-failure; false"
15751                }
15752            )];
15753            runner.state.config.verify.gate = vec![format!(
15754                "echo gate >> '{}'; {}",
15755                phases.display(),
15756                if gate_ok {
15757                    "true"
15758                } else {
15759                    "echo new-gate-failure; false"
15760                }
15761            )];
15762            runner.state.save().unwrap();
15763            runner.state = RunState::load(&runner.state.id).unwrap();
15764            runner.sync_to_base().await.unwrap();
15765            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15766            assert_ne!(head, old);
15767            // Reentry after Git updated but before verification was invalidated.
15768            runner.state = RunState::load(&runner.state.id).unwrap();
15769            runner.execute_graph().await.unwrap();
15770            assert_eq!(runner.state.gate_fixes.len(), 1);
15771            assert_eq!(runner.state.reviews.len(), 1);
15772            assert_eq!(runner.state.reviews[0].head, old);
15773            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
15774            assert_eq!(runner.state.verification_history.len(), 1);
15775            let history = &runner.state.verification_history[0];
15776            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
15777            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15778            assert!(history.archived_at.is_some());
15779            assert_eq!(history.fixes_spent, Some(1));
15780            assert_eq!(history.gate[0].output_tail, "old failure");
15781            let archive_events: Vec<_> = runner
15782                .state
15783                .events
15784                .iter()
15785                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
15786                .collect();
15787            assert_eq!(archive_events.len(), 1);
15788            assert!(archive_events[0].message.contains(&short(&old)));
15789            assert!(archive_events[0].message.contains(&short(&head)));
15790            assert!(
15791                archive_events[0]
15792                    .message
15793                    .contains("1 gate-fix round(s) already spent stay spent")
15794            );
15795            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
15796            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
15797            if e2e_ok {
15798                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15799                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
15800            } else {
15801                assert!(!runner.state.gate_ran);
15802            }
15803            assert_eq!(
15804                runner.state.status,
15805                if e2e_ok && gate_ok {
15806                    RunStatus::Ready
15807                } else {
15808                    RunStatus::Blocked
15809                }
15810            );
15811            if !e2e_ok || !gate_ok {
15812                let cause = runner.state.verification_blocking_cause().unwrap();
15813                assert!(cause.contains(&head));
15814                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
15815                // A later successful progress event must not become the cause.
15816                runner.state.event("land", "rebased successfully");
15817                let notice = crate::notices::run_ended(&runner.state).unwrap();
15818                assert!(!format!("{notice:?}").contains("rebased successfully"));
15819                assert!(runner.state.merge.is_none());
15820            }
15821            let before = std::fs::read_to_string(&phases).unwrap();
15822            runner.state.save().unwrap();
15823            runner.state = RunState::load(&runner.state.id).unwrap();
15824            runner.review_loop().await.unwrap();
15825            runner.gate().await.unwrap();
15826            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
15827            assert_eq!(runner.state.verification_history.len(), 1);
15828            let history = &runner.state.verification_history[0];
15829            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15830            assert!(history.archived_at.is_some());
15831            assert_eq!(history.fixes_spent, Some(1));
15832        }
15833    }
15834
15835    #[cfg(unix)]
15836    #[tokio::test]
15837    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
15838        let (tmp, mut runner, head) = resumed_gate_fixture().await;
15839        let marker = tmp.path().join("gate-ran");
15840        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
15841        runner.review_loop().await.unwrap();
15842        runner.gate().await.unwrap();
15843        assert_eq!(runner.state.status, RunStatus::Blocked);
15844        assert!(!marker.exists());
15845        assert!(runner.state.verification_history.is_empty());
15846        runner.state.gate_head = None;
15847        runner.gate().await.unwrap();
15848        assert!(marker.exists());
15849        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15850        assert_eq!(runner.state.verification_history.len(), 1);
15851        assert_eq!(runner.state.gate_fixes.len(), 1);
15852    }
15853
15854    #[cfg(unix)]
15855    #[tokio::test]
15856    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
15857        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
15858        runner.sync_to_base().await.unwrap();
15859        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15860        runner.state.reconcile_verification_head(&head);
15861        // Both crash-before-command and resource wait are unresolved evidence.
15862        for outcomes in [
15863            Vec::new(),
15864            vec![CommandOutcome {
15865                command: "true".into(),
15866                code: None,
15867                output_tail: "cache unavailable".into(),
15868                duration_ms: 0,
15869                resource_blocked: true,
15870            }],
15871        ] {
15872            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
15873                head: head.clone(),
15874                passed: false,
15875                outcomes,
15876            });
15877            runner.state.gate_ran = false;
15878            runner.state.gate.clear();
15879            runner.state.save().unwrap();
15880            runner.state = RunState::load(&runner.state.id).unwrap();
15881            runner.merge().await.unwrap();
15882            assert!(runner.state.merge.is_none());
15883            runner.review_loop().await.unwrap();
15884            runner.gate().await.unwrap();
15885            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
15886            assert!(runner.state.gate_allows_progress());
15887            assert_eq!(runner.state.gate_fixes.len(), 1);
15888            assert!(runner.state.merge.is_none());
15889        }
15890        // Moving the winner after a successful gate cannot publish an unchecked SHA.
15891        git::git(
15892            &runner.state.repo,
15893            &["commit", "--allow-empty", "-m", "external update"],
15894        )
15895        .await
15896        .unwrap();
15897        runner.merge().await.unwrap();
15898        assert!(runner.state.merge.is_none());
15899        assert_eq!(runner.state.status, RunStatus::Blocked);
15900    }
15901
15902    #[cfg(unix)]
15903    #[tokio::test]
15904    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
15905        for waiting_on_gate in [false, true] {
15906            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
15907            runner.sync_to_base().await.unwrap();
15908            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15909            runner.state.reconcile_verification_head(&head);
15910            if waiting_on_gate {
15911                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
15912                    head: head.clone(),
15913                    passed: true,
15914                    outcomes: Vec::new(),
15915                });
15916            }
15917            let cache = tmp.path().join("cache");
15918            runner.state.config.verify.e2e =
15919                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15920            runner.state.config.verify.gate = vec!["true".into()];
15921            runner.state.config.graph.timeout_resource_wait = 1;
15922            let owner =
15923                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
15924            let guard =
15925                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
15926                    crate::cache::AcquireOutcome::Acquired(g) => g,
15927                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
15928                };
15929            runner.review_loop().await.unwrap();
15930            runner.gate().await.unwrap();
15931            assert!(!runner.state.gate_ran);
15932            assert_eq!(runner.state.gate_fixes.len(), 1);
15933            assert!(runner.state.resource_block.is_some());
15934            runner.merge().await.unwrap();
15935            assert!(runner.state.merge.is_none());
15936            drop(guard);
15937            runner.state = RunState::load(&runner.state.id).unwrap();
15938            // The driver clears the previous pass's resource stop on resume.
15939            runner.state.resource_block = None;
15940            runner.review_loop().await.unwrap();
15941            runner.gate().await.unwrap();
15942            assert!(runner.state.gate_allows_progress());
15943            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15944            assert_eq!(runner.state.gate_fixes.len(), 1);
15945        }
15946    }
15947
15948    #[cfg(unix)]
15949    #[tokio::test]
15950    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
15951        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
15952        runner.sync_to_base().await.unwrap();
15953        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15954        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
15955        // Verification passes before pre_gate and fails on the mechanically changed head.
15956        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
15957        runner.review_loop().await.unwrap();
15958        runner.gate().await.unwrap();
15959        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15960        assert_ne!(before, after);
15961        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
15962        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
15963        assert!(!runner.state.gate_allows_progress());
15964        runner.merge().await.unwrap();
15965        assert!(runner.state.merge.is_none());
15966        assert!(
15967            runner
15968                .state
15969                .verification_history
15970                .iter()
15971                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
15972        );
15973
15974        // An incomplete review is still an independent hold on a changed head.
15975        runner.state.reviews[0].clean = false;
15976        runner.state.reviews[0].answered = 0;
15977        runner.state.config.verify.e2e = vec!["true".into()];
15978        runner.review_loop().await.unwrap();
15979        runner.gate().await.unwrap();
15980        runner.merge().await.unwrap();
15981        assert_eq!(runner.state.status, RunStatus::Blocked);
15982        assert!(runner.state.merge.is_none());
15983        assert!(
15984            runner
15985                .state
15986                .verification_blocking_cause()
15987                .unwrap()
15988                .contains("review blocked")
15989        );
15990    }
15991
15992    #[cfg(unix)]
15993    #[tokio::test]
15994    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
15995        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
15996        runner.state.gate_head = None;
15997        runner.state.config.verify.gate =
15998            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
15999        runner.review_loop().await.unwrap();
16000        runner.gate().await.unwrap();
16001        assert_ne!(
16002            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16003            old
16004        );
16005        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16006        assert!(!runner.state.gate_allows_progress());
16007        assert_eq!(runner.state.status, RunStatus::Blocked);
16008        runner.merge().await.unwrap();
16009        assert!(runner.state.merge.is_none());
16010    }
16011
16012    #[cfg(unix)]
16013    #[tokio::test]
16014    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16015        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16016        let mut old_review = review_round(true, 0, 1, 1, false, true);
16017        old_review.head = head.clone();
16018        old_review.verified_head = Some(head.clone());
16019        runner.state.reviews = vec![old_review];
16020        runner.state.gate = vec![CommandOutcome {
16021            command: "old gate".into(),
16022            code: None,
16023            output_tail: "timed out after 2400s".into(),
16024            duration_ms: 2_400_000,
16025            resource_blocked: false,
16026        }];
16027        runner.state.gate_ran = true;
16028        runner.state.gate_head = Some(head.clone());
16029        runner.state.land_approval = Some(crate::run::LandApproval {
16030            question: "old-head-approval".into(),
16031            head: head.clone(),
16032        });
16033        runner.state.authorize_base_sync(&task).unwrap();
16034        runner.sync_to_base().await.unwrap();
16035        assert!(!runner.state.base_sync_blocked());
16036        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16037        assert_ne!(head, fresh);
16038        assert!(runner.state.reviews.is_empty());
16039        assert!(!runner.state.gate_ran);
16040        let saved = &runner.state.base_sync_recovery.authorizations[0];
16041        assert_eq!(saved.reviews[0].head, head);
16042        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16043        assert!(saved.gate_ran);
16044        assert!(!saved.gate[0].ok());
16045        runner.review_loop().await.unwrap();
16046        runner.gate().await.unwrap();
16047        assert_eq!(runner.state.reviews[0].head, fresh);
16048        assert_eq!(
16049            runner.state.reviews[0].verified_head.as_deref(),
16050            Some(fresh.as_str())
16051        );
16052        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16053        assert!(runner.state.gate_ran);
16054        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16055        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16056        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16057        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16058        assert_eq!(
16059            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16060            "retained change"
16061        );
16062    }
16063
16064    #[cfg(unix)]
16065    #[tokio::test]
16066    async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16067        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16068        let queue = crate::queue::Queue::open();
16069        let mut owner = queue.get(&task).unwrap();
16070        owner.release_operator("test-resume", None);
16071        queue.put(&mut owner).unwrap();
16072        let generation = owner.resumption.as_ref().unwrap().generation;
16073        runner.state.task_instruction = Some(owner.instruction.clone());
16074        runner.state.driver_exited = true;
16075        runner.state.save().unwrap();
16076        let home = crate::run::home();
16077        let _claim = queue.claim(&task).unwrap();
16078        owner.resumption.as_mut().unwrap().at =
16079            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16080        queue.put(&mut owner).unwrap();
16081        assert!(
16082            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16083                .await
16084                .is_err(),
16085            "an approval from before the stop grants nothing"
16086        );
16087        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16088        queue.put(&mut owner).unwrap();
16089        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16090            .await
16091            .unwrap();
16092        assert!(
16093            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16094                .await
16095                .is_err()
16096        );
16097        runner.state = RunState::load(&runner.state.id).unwrap();
16098        assert_eq!(
16099            runner.state.base_sync_recovery.authorizations[0].generation,
16100            Some(generation)
16101        );
16102        runner.sync_to_base().await.unwrap();
16103        let repo = runner.state.repo.clone();
16104        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16105        git::git(&repo, &["checkout", "main"]).await.unwrap();
16106        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16107        git::git(&repo, &["add", "."]).await.unwrap();
16108        git::git(&repo, &["commit", "-m", "move base again"])
16109            .await
16110            .unwrap();
16111        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16112        git::git(&repo, &["checkout", "fix/retained"])
16113            .await
16114            .unwrap();
16115        runner.finish_after_tally().await.unwrap();
16116        assert_eq!(runner.state.status, RunStatus::Blocked);
16117        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16118        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16119        assert!(!runner.state.base_recovery_pending());
16120        assert!(!crate::handover::can_resume(&runner.state));
16121        assert!(!runner.state.gate_ran);
16122        // Even replaying the generation with a later timestamp grants nothing.
16123        owner.resumption.as_mut().unwrap().at = Timestamp::now();
16124        queue.put(&mut owner).unwrap();
16125        assert!(
16126            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16127                .await
16128                .is_err()
16129        );
16130        runner.state = RunState::load(&runner.state.id).unwrap();
16131        runner.sync_to_base().await.unwrap();
16132        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16133        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16134        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16135    }
16136
16137    #[cfg(unix)]
16138    #[tokio::test]
16139    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16140        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16141        runner.state.base_sync = None;
16142        runner.state.status = RunStatus::Blocked;
16143        let mut second = runner.state.candidates[0].clone();
16144        second.label = 'B';
16145        runner.state.candidates.push(second);
16146        assert!(runner.state.votes.is_empty());
16147        assert!(runner.state.deliberation.is_empty());
16148        runner.deliberate().await.unwrap();
16149        runner.vote().await.unwrap();
16150        assert_eq!(runner.state.status, RunStatus::Blocked);
16151        assert!(runner.state.votes.is_empty());
16152        assert!(runner.state.deliberation.is_empty());
16153    }
16154
16155    /// A minimal, git-free `Runner` at a given status — `settle_questions`
16156    /// reads nothing else off it.
16157    fn runner_at(status: RunStatus) -> Runner {
16158        let mut state = RunState::new(
16159            PathBuf::from("/nonexistent/repo"),
16160            "main".to_owned(),
16161            "deadbeef".to_owned(),
16162            "task".to_owned(),
16163            Config::default(),
16164        );
16165        // These fixtures share one home and save concurrently. Short clock
16166        // IDs can collide, making an ownership test read another test's run.
16167        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16168        state.id = format!(
16169            "20000101-000000-{:04x}",
16170            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16171        );
16172        state.status = status;
16173        Runner {
16174            state,
16175            roles: ResolvedRoles {
16176                implementers: Vec::new(),
16177                judges: Vec::new(),
16178                reviewers: Vec::new(),
16179                fixer: None,
16180                conductor: conductor(),
16181                implementer_roster: Vec::new(),
16182                judge_roster: Vec::new(),
16183                reviewer_roster: Vec::new(),
16184            },
16185            sem: Arc::new(Semaphore::new(1)),
16186            pause: Pause::new(),
16187            interrupt: Pause::new(),
16188        }
16189    }
16190
16191    #[tokio::test]
16192    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16193        crate::run::pin_test_home();
16194        let mut runner = runner_at(RunStatus::Gating);
16195        runner.state.save().unwrap();
16196        let id = runner.state.id.clone();
16197        let passes = runner.state.passes.len();
16198        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16199        let mut claiming = Box::pin(runner.claim_driver());
16200        std::future::poll_fn(|cx| {
16201            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16202            std::task::Poll::Ready(())
16203        })
16204        .await;
16205        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16206        drop(owner);
16207        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16208            .await
16209            .unwrap()
16210            .unwrap();
16211        assert_eq!(execution.runner.state.passes.len(), passes + 1);
16212        assert!(execution.runner.state.resource_block.is_none());
16213        assert_eq!(
16214            execution.runner.state.liveness(false),
16215            crate::run::Liveness::Live
16216        );
16217        drop(execution);
16218        assert!(RunState::load(&id).unwrap().driver_exited);
16219    }
16220
16221    #[tokio::test]
16222    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16223        crate::run::pin_test_home();
16224        let mut runner = runner_at(RunStatus::Gating);
16225        runner.state.save().unwrap();
16226        // run.json can commit, but the subsequent lease metadata write fails.
16227        // Cleanup must run after releasing the short ownership lock.
16228        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16229        assert!(runner.claim_driver().await.is_err());
16230        assert!(runner.state.driver_exited);
16231        assert_eq!(runner.state.driver_pid, None);
16232        let disk = RunState::load(&runner.state.id).unwrap();
16233        assert!(disk.driver_exited);
16234        assert_eq!(disk.driver_pid, None);
16235        assert_eq!(disk.status, RunStatus::Gating);
16236    }
16237
16238    #[tokio::test]
16239    async fn an_early_graph_error_releases_its_driver() {
16240        crate::run::pin_test_home();
16241        let mut runner = runner_at(RunStatus::Blocked);
16242        runner.state.config.graph.selected_implementer = Some("selected".into());
16243        runner.state.save().unwrap();
16244        assert!(runner.execute().await.is_err());
16245        let disk = RunState::load(&runner.state.id).unwrap();
16246        assert!(disk.driver_exited);
16247        assert_eq!(disk.driver_pid, None);
16248        assert_eq!(disk.status, RunStatus::Blocked);
16249    }
16250
16251    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16252        crate::run::MergeRepair {
16253            question: "repair-question".into(),
16254            head: "old-head".into(),
16255            pr: "https://github.com/o/r/pull/42".into(),
16256            reviews: vec![review_round(false, 1, 1, 1, false, true)],
16257            gate: vec![outcome(Some(0), false)],
16258            gate_head: Some("old-head".into()),
16259            gate_fixes: Vec::new(),
16260            pre_gate: Vec::new(),
16261            phase,
16262            failure: None,
16263        }
16264    }
16265
16266    #[tokio::test]
16267    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16268        crate::run::pin_test_home();
16269        let mut runner = runner_at(RunStatus::Landing);
16270        runner.state.merge_repairs.push(merge_repair_request(
16271            crate::run::MergeRepairPhase::Requested,
16272        ));
16273        runner.state.save().unwrap();
16274        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16275        assert!(
16276            runner
16277                .execute()
16278                .await
16279                .unwrap_err()
16280                .to_string()
16281                .contains("already running")
16282        );
16283        assert_eq!(
16284            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16285            crate::run::MergeRepairPhase::Requested
16286        );
16287        drop(claim);
16288        let mut disk = runner.state.clone();
16289        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16290        disk.save().unwrap();
16291        assert!(
16292            runner
16293                .execute()
16294                .await
16295                .unwrap_err()
16296                .to_string()
16297                .contains("decision changed")
16298        );
16299        assert_eq!(
16300            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16301            crate::run::MergeRepairPhase::Complete
16302        );
16303    }
16304
16305    #[test]
16306    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16307        crate::run::pin_test_home();
16308        let mut runner = runner_at(RunStatus::Landing);
16309        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16310        runner.state.reviews = request.reviews.clone();
16311        runner.state.gate = request.gate.clone();
16312        runner.state.gate_ran = true;
16313        runner.state.land_approval = Some(crate::run::LandApproval {
16314            question: "old".into(),
16315            head: "old-head".into(),
16316        });
16317        runner.state.merge_repairs.push(request);
16318        runner.begin_merge_repair_review(0).unwrap();
16319        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16320        assert!(!runner.state.gate_status().ok());
16321        assert!(runner.state.land_approval.is_none());
16322        let disk = RunState::load(&runner.state.id).unwrap();
16323        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16324        assert!(disk.merge_repairs[0].gate[0].ok());
16325        assert_eq!(
16326            disk.merge_repairs[0].phase,
16327            crate::run::MergeRepairPhase::Reviewing
16328        );
16329        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16330        assert!(disk.reviews.is_empty());
16331    }
16332
16333    #[tokio::test]
16334    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16335        crate::run::pin_test_home();
16336        for phase in [
16337            crate::run::MergeRepairPhase::Fixing,
16338            crate::run::MergeRepairPhase::Requested,
16339        ] {
16340            let mut runner = runner_at(RunStatus::Landing);
16341            runner.state.config.graph.land_rounds = 0;
16342            runner.state.merge_repairs.push(merge_repair_request(phase));
16343            runner.state.save().unwrap();
16344            assert!(!runner.repair_merge_review().await.unwrap());
16345            assert_eq!(runner.state.status, RunStatus::Blocked);
16346            assert_eq!(
16347                runner.state.merge_repairs[0].phase,
16348                crate::run::MergeRepairPhase::Failed
16349            );
16350            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16351            assert!(
16352                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16353                    "interrupted"
16354                } else {
16355                    "budget exhausted"
16356                })
16357            );
16358            assert!(runner.state.merge.is_none());
16359        }
16360    }
16361
16362    /// `park_here` folding in the reason `Pause::park_because` recorded -
16363    /// this is what lets an operator reading a run's events tell an
16364    /// interrupt-driven park from an ordinary shutdown park.
16365    #[test]
16366    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16367        crate::run::pin_test_home();
16368        let mut runner = runner_at(RunStatus::Implementing);
16369        let interrupt = Pause::new();
16370        runner.watch_interrupt(interrupt.clone());
16371
16372        interrupt.park_because("task a1b2 asked to run first");
16373
16374        assert!(runner.park_here().expect("park_here"));
16375        assert!(runner.state.parked);
16376        let last = runner.state.events.last().expect("a park event");
16377        assert_eq!(last.node, "park");
16378        assert!(
16379            last.message.contains("task a1b2 asked to run first"),
16380            "expected the interrupt reason in {:?}",
16381            last.message
16382        );
16383    }
16384
16385    /// The cause is stored on the pass when it parks, and the first cause a
16386    /// shared pause was given is the one that stays.
16387    #[test]
16388    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16389        crate::run::pin_test_home();
16390        let mut runner = runner_at(RunStatus::Implementing);
16391        runner.state.begin_pass();
16392        let shutdown = Pause::new();
16393        runner.on_pause(shutdown.clone());
16394        let upgrade = ParkCause::Upgrade {
16395            from: "0.121.0".to_owned(),
16396            to: Some("0.121.1".to_owned()),
16397        };
16398        shutdown.park_for(upgrade.clone());
16399        shutdown.park_for(ParkCause::Operator);
16400
16401        assert!(runner.park_here().expect("park_here"));
16402        assert_eq!(
16403            runner.state.passes[0].end,
16404            Some(crate::run::PassEnd::Parked { cause: upgrade })
16405        );
16406        let last = runner.state.events.last().expect("a park event");
16407        assert!(last.message.contains("0.121.0"), "{}", last.message);
16408    }
16409
16410    #[tokio::test]
16411    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16412        crate::run::pin_test_home();
16413        let mut runner = runner_at(RunStatus::Reviewing);
16414        runner.state.save().unwrap();
16415        let mut execution = runner.claim_driver().await.expect("first execution");
16416        assert_eq!(execution.runner.state.passes.len(), 1);
16417        execution
16418            .driver
16419            .finish(&mut execution.runner.state)
16420            .await
16421            .unwrap();
16422    }
16423
16424    #[tokio::test]
16425    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
16426        crate::run::pin_test_home();
16427        let mut runner = runner_at(RunStatus::Reviewing);
16428        runner.state.config.agents = vec![conductor()];
16429        runner.state.config.graph.timeout_resource_wait = 0;
16430        runner.state.save().unwrap();
16431        let id = runner.state.id.clone();
16432        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16433        assert!(runner.claim_driver().await.is_err());
16434        drop(owner);
16435        drop(runner);
16436        let mut resumed = Runner::resume(&id).unwrap();
16437        assert_eq!(resumed.state.driver_claimed, Some(false));
16438        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
16439        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
16440        let mut execution = resumed
16441            .claim_driver()
16442            .await
16443            .expect("retry initial claim after reload");
16444        assert_eq!(execution.runner.state.driver_claimed, Some(true));
16445        assert_eq!(execution.runner.state.passes.len(), 1);
16446        execution
16447            .driver
16448            .finish(&mut execution.runner.state)
16449            .await
16450            .unwrap();
16451    }
16452
16453    #[tokio::test]
16454    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
16455        crate::run::pin_test_home();
16456        let mut runner = runner_at(RunStatus::Reviewing);
16457        runner.state.config.agents = vec![conductor()];
16458        runner.state.save().unwrap();
16459        let id = runner.state.id.clone();
16460        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
16461        let mut resumed = Runner::resume(&id).unwrap();
16462        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
16463        assert!(resumed.claim_driver().await.is_err());
16464        assert!(RunState::load(&id).unwrap().passes.is_empty());
16465        drop(lease);
16466        let mut execution = resumed.claim_driver().await.unwrap();
16467        execution
16468            .driver
16469            .finish(&mut execution.runner.state)
16470            .await
16471            .unwrap();
16472    }
16473
16474    #[tokio::test]
16475    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
16476        crate::run::pin_test_home();
16477        let mut runner = runner_at(RunStatus::Reviewing);
16478        let state = &mut runner.state;
16479        state.config.agents = vec![conductor()];
16480        state.driver_claimed = None;
16481        state.driver_pid = Some(std::process::id());
16482        state.driver_started_at = None;
16483        state.begin_pass();
16484        state.save().unwrap();
16485        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
16486        std::fs::write(
16487            state.dir().join("driver.json"),
16488            serde_json::to_vec(&serde_json::json!({
16489                "pid": state.driver_pid, "started_at": null,
16490                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
16491            }))
16492            .unwrap(),
16493        )
16494        .unwrap();
16495        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
16496        drop(lease);
16497        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
16498        let mut resumed = Runner::resume(&state.id).unwrap();
16499        assert!(!resumed.state.driver_exited);
16500        let mut execution = resumed
16501            .claim_driver()
16502            .await
16503            .expect("lease proves previous execution ended");
16504        assert_eq!(execution.runner.state.passes.len(), 2);
16505        execution
16506            .driver
16507            .finish(&mut execution.runner.state)
16508            .await
16509            .unwrap();
16510    }
16511
16512    #[tokio::test]
16513    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16514        crate::run::pin_test_home();
16515        let mut runner = runner_at(RunStatus::Reviewing);
16516        let mut json = serde_json::to_value(&runner.state).unwrap();
16517        json.as_object_mut().unwrap().remove("driver_claimed");
16518        runner.state = serde_json::from_value(json).unwrap();
16519        assert_eq!(runner.state.driver_claimed, None);
16520        runner.state.save().unwrap();
16521        let mut execution = runner
16522            .claim_driver()
16523            .await
16524            .expect("the execution lease, not a missing marker, excludes a second driver");
16525        assert_eq!(execution.runner.state.passes.len(), 1);
16526        execution
16527            .driver
16528            .finish(&mut execution.runner.state)
16529            .await
16530            .unwrap();
16531    }
16532
16533    #[tokio::test]
16534    async fn driver_admission_refuses_a_proven_live_driver() {
16535        crate::run::pin_test_home();
16536        let mut runner = runner_at(RunStatus::Reviewing);
16537        let pid = std::process::id();
16538        runner.state.driver_pid = Some(pid);
16539        runner.state.driver_started_at = crate::proc::process_started_at(pid);
16540        assert!(runner.state.driver_started_at.is_some());
16541        runner.state.save().unwrap();
16542        let error = runner
16543            .claim_driver()
16544            .await
16545            .err()
16546            .expect("a live driver must refuse");
16547        assert!(error.to_string().contains("live driver"), "{error:#}");
16548        let saved = RunState::load(&runner.state.id).unwrap();
16549        assert_eq!(saved.driver_pid, Some(pid));
16550        assert_eq!(saved.status, RunStatus::Reviewing);
16551        assert!(saved.passes.is_empty());
16552    }
16553
16554    #[test]
16555    fn parking_resource_halt_persists_the_pass_once() {
16556        crate::run::pin_test_home();
16557        let mut runner = runner_at(RunStatus::Reviewing);
16558        runner.state.begin_pass();
16559        runner.state.resource_block = Some(crate::run::ResourceBlock {
16560            node: "fix".into(),
16561            kind: "shared build cache".into(),
16562            detail: "cancelled".into(),
16563            at: Timestamp::now(),
16564            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16565        });
16566        let cause = ParkCause::Upgrade {
16567            from: "old".into(),
16568            to: Some("new".into()),
16569        };
16570        runner.pause.park_for(cause.clone());
16571        // Both direct fixer exits and graph resource boundaries must complete
16572        // parking, rather than returning a stale Reviewing record.
16573        runner.stop_for_resource().unwrap();
16574        assert!(runner.resource_halt().unwrap());
16575        assert!(runner.park_here().unwrap());
16576        let saved = RunState::load(&runner.state.id).unwrap();
16577        assert!(saved.parked);
16578        assert_eq!(saved.status, RunStatus::Reviewing);
16579        assert!(saved.passes[0].ended_at.is_some());
16580        assert_eq!(
16581            saved.passes[0].end,
16582            Some(crate::run::PassEnd::Parked {
16583                cause: cause.clone()
16584            })
16585        );
16586        assert_eq!(
16587            saved.resource_block.unwrap().cause,
16588            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16589        );
16590        assert_eq!(
16591            saved
16592                .events
16593                .iter()
16594                .filter(|event| event.node == "park")
16595                .count(),
16596            1
16597        );
16598    }
16599
16600    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
16601    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
16602    /// cleared) must not make a *different* run - one only watching its own,
16603    /// unshared interrupt `Pause` - see itself as parked. If a future change
16604    /// ever collapsed these back into one handle, the interrupt scheduler
16605    /// would park every run for the rest of the daemon's life, not just the
16606    /// one it meant to interrupt.
16607    #[test]
16608    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16609        crate::run::pin_test_home();
16610        let mut runner = runner_at(RunStatus::Implementing);
16611        let shutdown = Pause::new();
16612        runner.on_pause(shutdown.clone());
16613        let interrupt = Pause::new();
16614        runner.watch_interrupt(interrupt.clone());
16615
16616        // Nobody has asked for anything yet.
16617        assert!(!runner.park_here().expect("park_here"));
16618        assert!(!runner.state.parked);
16619
16620        // Only the interrupt handle fires; the shutdown handle stays clear.
16621        interrupt.park_because("test");
16622        assert!(!shutdown.parked());
16623        assert!(runner.park_here().expect("park_here"));
16624    }
16625
16626    /// The property every prior attempt at this feature failed to pin down:
16627    /// asking a run to park while one of its nodes has a real, in-flight
16628    /// async operation running (an agent call, in production) must not cut
16629    /// that operation short. `park_here` is only ever consulted *between*
16630    /// `execute`'s node calls - see its own doc - so nothing inside a node
16631    /// can observe a park request until the node itself returns. This proves
16632    /// that structurally, with real `tokio` concurrency and a channel
16633    /// handshake (never a sleep, which would only prove "usually", not
16634    /// "cannot"): the "node" below reports that it has genuinely started,
16635    /// and only then is the park requested; the node still has to be told to
16636    /// finish before `park_here` is ever called, exactly mirroring every
16637    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
16638    /// in `execute`.
16639    #[tokio::test]
16640    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16641        crate::run::pin_test_home();
16642        let mut runner = runner_at(RunStatus::Implementing);
16643        let interrupt = Pause::new();
16644        runner.watch_interrupt(interrupt.clone());
16645
16646        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16647        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16648
16649        // Stands in for one node's in-flight agent call: it proves it has
16650        // genuinely started, then blocks - exactly as a spawned CLI process
16651        // does - until told to finish.
16652        let node = async move {
16653            started_tx.send(()).expect("send started");
16654            finish_rx.await.expect("recv finish");
16655            "node finished"
16656        };
16657
16658        let interrupter = async move {
16659            started_rx.await.expect("recv started");
16660            // The call is now genuinely in flight. Ask it to park.
16661            interrupt.park_because("higher-priority task waiting");
16662            // Nothing the node does can observe this yet - there is no
16663            // check inside it, by construction - so let the executor run
16664            // anything pending and then let the node finish on its own.
16665            tokio::task::yield_now().await;
16666            finish_tx.send(()).expect("send finish");
16667        };
16668
16669        let (node_result, ()) = tokio::join!(node, interrupter);
16670        assert_eq!(
16671            node_result, "node finished",
16672            "the in-flight call ran to completion"
16673        );
16674
16675        // Only now, at the boundary the real `execute` would check right
16676        // after this node, does the park take effect.
16677        assert!(runner.park_here().expect("park_here"));
16678        assert!(runner.state.parked);
16679    }
16680
16681    /// A run parked mid-competition carries every field it had accumulated
16682    /// through the exact same disk round-trip an ordinary resume uses -
16683    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
16684    /// Nothing about parking for an interrupt is a special case of that path;
16685    /// this is what proves it rather than assuming it.
16686    #[test]
16687    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16688        crate::run::pin_test_home();
16689        let mut runner = runner_at(RunStatus::Judging);
16690        // `Runner::resume` re-resolves roles from the saved config, which
16691        // refuses an empty roster - give it the same minimal one `conductor`
16692        // itself uses.
16693        runner.state.config.agents = vec![conductor()];
16694        runner.state.candidates = vec![Candidate {
16695            index: 0,
16696            label: 'A',
16697            agent: "alpha".to_owned(),
16698            branch: "magi/x/A".to_owned(),
16699            worktree: PathBuf::from("/nonexistent/worktree"),
16700            summary: "did the thing".to_owned(),
16701            stat: "1 file changed".to_owned(),
16702            files: 1,
16703            commits: 1,
16704            empty: false,
16705            failed: None,
16706            verified_noop: None,
16707            duration_ms: 1234,
16708            folded: false,
16709        }];
16710        let run_id = runner.state.id.clone();
16711
16712        let interrupt = Pause::new();
16713        runner.watch_interrupt(interrupt.clone());
16714        interrupt.park_because("task c3d4 asked to run first");
16715        assert!(runner.park_here().expect("park_here"));
16716
16717        let resumed = Runner::resume(&run_id).expect("resume");
16718        assert_eq!(resumed.state.candidates.len(), 1);
16719        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16720        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16721        assert_eq!(resumed.state.status, runner.state.status);
16722        assert!(
16723            resumed.state.parked,
16724            "still parked until `execute` actually walks the graph again"
16725        );
16726        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16727    }
16728
16729    /// A fresh open question on `run`, stored and handed back for assertions.
16730    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16731        let mut q = ask::Question::new(
16732            run.to_owned(),
16733            "implement".to_owned(),
16734            "impl-A".to_owned(),
16735            "Which storage backend should the cache use?".to_owned(),
16736            String::new(),
16737            vec!["SQLite".to_owned(), "Redis".to_owned()],
16738        );
16739        store.put(&mut q).unwrap();
16740        q
16741    }
16742
16743    #[test]
16744    fn a_failed_runs_open_question_is_abandoned() {
16745        ask_test_home();
16746        let store = ask::Questions::open();
16747        let mut runner = runner_at(RunStatus::Failed);
16748        let run = runner.state.id.clone();
16749        let q = ask_open_question(&store, &run);
16750
16751        runner.settle_questions();
16752
16753        let back = store.get(&q.id).unwrap();
16754        assert!(
16755            !back.status.open(),
16756            "the seat that asked died with the run; nobody is left to read an answer"
16757        );
16758        assert!(
16759            back.detail.contains(&run) && back.detail.contains("failed"),
16760            "the reason names what the run became, not just that it is gone: {}",
16761            back.detail
16762        );
16763    }
16764
16765    #[test]
16766    fn a_merged_runs_open_question_is_abandoned_too() {
16767        ask_test_home();
16768        let store = ask::Questions::open();
16769        // A run that finishes cleanly still leaves nobody to read an answer -
16770        // this is not only a failure-path cleanup.
16771        for status in [RunStatus::Merged, RunStatus::Ready] {
16772            let mut runner = runner_at(status);
16773            let run = runner.state.id.clone();
16774            let q = ask_open_question(&store, &run);
16775
16776            runner.settle_questions();
16777
16778            let back = store.get(&q.id).unwrap();
16779            assert!(
16780                !back.status.open(),
16781                "{status:?} run's question must not outlive the run"
16782            );
16783        }
16784    }
16785
16786    #[test]
16787    fn a_still_resumable_runs_open_question_is_left_alone() {
16788        ask_test_home();
16789        let store = ask::Questions::open();
16790        // `Blocked` and `Stalled` can still be resumed — the candidates, the
16791        // review round and the seat sessions are all still on disk — so a
16792        // question asked mid-round may yet get a real answer from a real
16793        // resume. Sweeping it here would be exactly the failure mode this
16794        // whole feature exists to avoid on the other side.
16795        for status in [RunStatus::Blocked, RunStatus::Stalled] {
16796            let mut runner = runner_at(status);
16797            let run = runner.state.id.clone();
16798            let q = ask_open_question(&store, &run);
16799
16800            runner.settle_questions();
16801
16802            let back = store.get(&q.id).unwrap();
16803            assert!(
16804                back.status.open(),
16805                "{status:?} is still alive; the question must still be waiting"
16806            );
16807        }
16808    }
16809
16810    #[test]
16811    fn settle_questions_never_touches_an_already_answered_question() {
16812        ask_test_home();
16813        let store = ask::Questions::open();
16814        let mut runner = runner_at(RunStatus::Failed);
16815        let run = runner.state.id.clone();
16816        let mut q = ask_open_question(&store, &run);
16817        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16818            .unwrap();
16819        store.put(&mut q).unwrap();
16820
16821        // Called twice, the way a crash-recovered daemon reclaim and the
16822        // graph's own cleanup both can for the same run — `abandon_for_run`
16823        // only ever touches what is still open, so this must be inert both
16824        // times, not merely the second.
16825        runner.settle_questions();
16826        runner.settle_questions();
16827
16828        let back = store.get(&q.id).unwrap();
16829        assert_eq!(
16830            back.status,
16831            ask::QuestionStatus::Answered,
16832            "a real answer is a decision on record, never overwritten by a sweep"
16833        );
16834    }
16835
16836    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
16837    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
16838    /// without merging, whose winner is still the operator's answer to read.
16839    /// Nothing previously called `fold_run` itself with a real `tally`, so
16840    /// this is the first test to pin down the one distinction the whole
16841    /// automatic-fold feature depends on: the winner's worktree and branch
16842    /// must survive, everything else sharing the run's worktree bay - a
16843    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
16844    /// second sweep treats every non-winner directory under the bay alike -
16845    /// must not.
16846    #[tokio::test]
16847    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16848        crate::run::pin_test_home();
16849        let tmp = tempfile::tempdir().expect("tempdir");
16850        let repo = tmp.path().join("repo");
16851        std::fs::create_dir_all(&repo).unwrap();
16852        init_repo(&repo);
16853
16854        let mut config = Config::default();
16855        config.graph.worktree_root = Some(tmp.path().join("wt"));
16856
16857        let mut state = RunState::new(
16858            repo.clone(),
16859            "main".to_owned(),
16860            "deadbeef".to_owned(),
16861            "task".to_owned(),
16862            config,
16863        );
16864        let root = state.worktree_root();
16865        let wt_a = root.join("cand-A");
16866        let wt_b = root.join("cand-B");
16867        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16868            .await
16869            .expect("worktree A");
16870        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16871            .await
16872            .expect("worktree B");
16873
16874        state.candidates = vec![
16875            Candidate {
16876                index: 0,
16877                label: 'A',
16878                agent: "alpha".to_owned(),
16879                branch: "magi/x/A".to_owned(),
16880                worktree: wt_a.clone(),
16881                summary: String::new(),
16882                stat: String::new(),
16883                files: 0,
16884                commits: 0,
16885                empty: false,
16886                failed: None,
16887                verified_noop: None,
16888                duration_ms: 0,
16889                folded: false,
16890            },
16891            Candidate {
16892                index: 1,
16893                label: 'B',
16894                agent: "beta".to_owned(),
16895                branch: "magi/x/B".to_owned(),
16896                worktree: wt_b.clone(),
16897                summary: String::new(),
16898                stat: String::new(),
16899                files: 0,
16900                commits: 0,
16901                empty: false,
16902                failed: None,
16903                verified_noop: None,
16904                duration_ms: 0,
16905                folded: false,
16906            },
16907        ];
16908        state.tally = Some(Tally {
16909            first_choice: BTreeMap::from([('A', 1)]),
16910            borda: BTreeMap::new(),
16911            winner: 'A',
16912            rankings: 1,
16913            unanimous_initial: true,
16914            deliberated: false,
16915            changed_votes: 0,
16916            unanimous_final: true,
16917            tie_break: None,
16918            judges: 1,
16919            present: 1,
16920            quorum: 1,
16921            met_quorum: true,
16922            uncontested: None,
16923        });
16924        state.status = RunStatus::Ready;
16925
16926        fold_run(&mut state, false, &crate::run::home())
16927            .await
16928            .expect("fold_run");
16929
16930        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
16931        assert!(
16932            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16933            "the unmerged winner's branch survives"
16934        );
16935        assert!(
16936            !state.candidates[0].folded,
16937            "the winner is not marked folded"
16938        );
16939
16940        assert!(!wt_b.exists(), "the loser's worktree is removed");
16941        assert!(
16942            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
16943            "the loser's branch is removed"
16944        );
16945        assert!(state.candidates[1].folded, "the loser is marked folded");
16946    }
16947
16948    #[tokio::test]
16949    #[cfg(unix)]
16950    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
16951        use std::os::fd::AsRawFd as _;
16952        use std::time::Duration;
16953
16954        let tmp = tempfile::tempdir().unwrap();
16955        let repo = tmp.path().join("repo");
16956        std::fs::create_dir_all(&repo).unwrap();
16957        init_repo(&repo);
16958        let common = repo.join(".git");
16959        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
16960        let home = tmp.path().join("home");
16961        let mut config = Config::default();
16962        config.coordination = crate::config::test_coordination();
16963        config.graph.worktree_root = Some(tmp.path().join("wt"));
16964        config.graph.timeout_resource_wait = 1;
16965        let mut state = RunState::new(
16966            repo.clone(),
16967            "main".into(),
16968            "deadbeef".into(),
16969            "task".into(),
16970            config,
16971        );
16972        let root = state.worktree_root();
16973        let candidate = root.join("cand-A");
16974        let reviewer = root.join("review-1");
16975        let branch = state.branch_for('A');
16976        git::worktree_add_branch(&repo, &candidate, &branch, "main")
16977            .await
16978            .unwrap();
16979        git::worktree_add_detached(&repo, &reviewer, "main")
16980            .await
16981            .unwrap();
16982        git::acquire_worktree_config(&repo).await.unwrap();
16983        state.enabled_worktree_config = true;
16984        state.status = RunStatus::Ready;
16985        state.candidates.push(Candidate {
16986            index: 0,
16987            label: 'A',
16988            agent: "implement".into(),
16989            branch: branch.clone(),
16990            worktree: candidate.clone(),
16991            summary: String::new(),
16992            stat: String::new(),
16993            files: 0,
16994            commits: 0,
16995            empty: false,
16996            failed: None,
16997            verified_noop: None,
16998            duration_ms: 0,
16999            folded: false,
17000        });
17001        state.save_under(&home).unwrap();
17002        let config_before = std::fs::read(common.join("config")).unwrap();
17003        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17004            .await
17005            .unwrap();
17006        // A separate OS handle bypasses the launcher's local gate, exactly as
17007        // an external updater does, and must still prevent all folding work.
17008        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17009        assert_eq!(
17010            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17011            0
17012        );
17013        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17014        assert!(
17015            tokio::time::timeout(Duration::from_millis(200), &mut fold)
17016                .await
17017                .is_err()
17018        );
17019        assert!(candidate.exists());
17020        assert!(reviewer.exists());
17021        assert!(git::branch_exists(&repo, &branch).await.unwrap());
17022        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17023        assert_eq!(
17024            git::git(&repo, &["worktree", "list", "--porcelain"])
17025                .await
17026                .unwrap(),
17027            worktrees_before
17028        );
17029        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17030            .await
17031            .unwrap()
17032            .unwrap_err();
17033        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17034        drop(fold);
17035        assert!(candidate.exists() && reviewer.exists());
17036        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17037        assert_eq!(
17038            state.resource_block.as_ref().unwrap().kind,
17039            "repository coordination"
17040        );
17041        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17042        drop(external);
17043        let mut fold = Box::pin(fold_run(&mut state, true, &home));
17044        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17045            .await
17046            .unwrap()
17047            .unwrap();
17048        drop(fold);
17049        assert!(removed.contains(&branch));
17050        assert!(!root.exists());
17051        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17052        assert!(state.candidates[0].folded);
17053        assert!(!state.enabled_worktree_config);
17054        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17055        let enabled = git::git_raw(
17056            &repo,
17057            &["config", "--local", "--get", "extensions.worktreeConfig"],
17058        )
17059        .await
17060        .unwrap();
17061        assert!(!enabled.ok(), "the last config reference was released");
17062    }
17063
17064    /// A branch handed to a later run is that run's (and its pull request's):
17065    /// folding the run that released it must not delete it.
17066    #[tokio::test]
17067    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17068        let tmp = tempfile::tempdir().expect("tempdir");
17069        let repo = tmp.path().join("repo");
17070        std::fs::create_dir_all(&repo).unwrap();
17071        init_repo(&repo);
17072        let home = tmp.path().join("home");
17073
17074        let mut config = Config::default();
17075        config.graph.worktree_root = Some(tmp.path().join("wt"));
17076        let mut state = RunState::new(
17077            repo.clone(),
17078            "main".to_owned(),
17079            "deadbeef".to_owned(),
17080            "task".to_owned(),
17081            config,
17082        );
17083        // The worktree is already gone (released); the branch survives.
17084        git::git(&repo, &["branch", "magi/x/A", "main"])
17085            .await
17086            .expect("branch");
17087        state.candidates = vec![Candidate {
17088            index: 0,
17089            label: 'A',
17090            agent: "alpha".to_owned(),
17091            branch: "magi/x/A".to_owned(),
17092            worktree: state.worktree_root().join("cand-A"),
17093            summary: String::new(),
17094            stat: String::new(),
17095            files: 0,
17096            commits: 0,
17097            empty: false,
17098            failed: None,
17099            verified_noop: None,
17100            duration_ms: 0,
17101            folded: true,
17102        }];
17103        state.released_to = Some("20260901-000000-new1".to_owned());
17104        state.released_branches = vec!["magi/x/A".to_owned()];
17105
17106        fold_run(&mut state, true, &home).await.expect("fold_run");
17107        state.status = RunStatus::Merged;
17108        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17109        state.merge = Some(crate::run::MergeOutcome {
17110            mode: MergeMode::Pr,
17111            ok: true,
17112            detail: "confirmed merged".into(),
17113            empty: false,
17114        });
17115        fold_run(&mut state, true, &home)
17116            .await
17117            .expect("merged fold");
17118
17119        assert!(
17120            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17121            "the handed-over branch survives a fold"
17122        );
17123    }
17124
17125    /// A winner with nothing ahead of the base is caught before `gh` is ever
17126    /// asked for a pull request, and the message carries what the task's
17127    /// references resolved to.
17128    #[tokio::test]
17129    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17130        let tmp = tempfile::tempdir().expect("tempdir");
17131        let repo = tmp.path().join("repo");
17132        std::fs::create_dir_all(&repo).unwrap();
17133        init_repo(&repo);
17134        let run = |args: &[&str]| {
17135            let out = std::process::Command::new("git")
17136                .quiet()
17137                .args(args)
17138                .current_dir(&repo)
17139                .output()
17140                .expect("spawn git");
17141            assert!(out.status.success(), "git {args:?}");
17142        };
17143        run(&["branch", "magi/x/A"]);
17144        run(&["checkout", "-q", "-b", "magi/x/B"]);
17145        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17146        run(&["add", "-A"]);
17147        run(&["commit", "-q", "-m", "work"]);
17148        run(&["checkout", "-q", "main"]);
17149
17150        // A pull request is compared against the remote's base, so the
17151        // fixture needs one. Before it exists nothing can be read, and the
17152        // answer must be "not empty".
17153        let probe = RunState::new(
17154            repo.clone(),
17155            "main".to_owned(),
17156            "deadbeef".to_owned(),
17157            "task".to_owned(),
17158            Config::default(),
17159        );
17160        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17161        let bare = tmp.path().join("origin.git");
17162        let out = std::process::Command::new("git")
17163            .quiet()
17164            .args(["init", "-q", "--bare"])
17165            .arg(&bare)
17166            .output()
17167            .expect("spawn git");
17168        assert!(out.status.success(), "git init --bare");
17169        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17170        run(&["push", "-q", "origin", "main"]);
17171
17172        let mut state = RunState::new(
17173            repo.clone(),
17174            "main".to_owned(),
17175            "deadbeef".to_owned(),
17176            "task".to_owned(),
17177            Config::default(),
17178        );
17179        state.seeds = vec![refs::Seed {
17180            token: "magi/27b2/A".to_owned(),
17181            kind: refs::SeedKind::Unresolved,
17182            sha: String::new(),
17183            branch: true,
17184            detail: "no branch or commit named magi/27b2/A".to_owned(),
17185        }];
17186
17187        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17188        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17189        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17190        let detail = empty_candidate_detail(&state, "main");
17191        assert!(detail.starts_with("empty candidate"), "{detail}");
17192        assert!(!detail.contains("magi/27b2/A"), "{detail}");
17193        assert!(detail.contains("--start-from"), "{detail}");
17194    }
17195
17196    /// `status == Ready` used to be read as "this is the harmless
17197    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
17198    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
17199    /// run's PR was closed without merging — and reentering `merge` with
17200    /// `mode` still `Pr` does not know the difference, so it pushed and
17201    /// opened a second pull request. `mode == Local` reproduces the same
17202    /// blind spot without a network call: reentry must not attempt another
17203    /// git merge once this node has already recorded an outcome.
17204    #[tokio::test]
17205    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17206        let tmp = tempfile::tempdir().expect("tempdir");
17207        let repo = tmp.path().join("repo");
17208        std::fs::create_dir_all(&repo).unwrap();
17209        init_repo(&repo);
17210
17211        let mut config = Config::default();
17212        config.merge.mode = MergeMode::Local;
17213
17214        let mut state = RunState::new(
17215            repo.clone(),
17216            "main".to_owned(),
17217            "deadbeef".to_owned(),
17218            "task".to_owned(),
17219            config,
17220        );
17221        state.candidates = vec![Candidate {
17222            index: 0,
17223            label: 'A',
17224            agent: "alpha".to_owned(),
17225            branch: "does-not-exist".to_owned(),
17226            worktree: repo.clone(),
17227            summary: String::new(),
17228            stat: String::new(),
17229            files: 0,
17230            commits: 0,
17231            empty: false,
17232            failed: None,
17233            verified_noop: None,
17234            duration_ms: 0,
17235            folded: false,
17236        }];
17237        state.tally = Some(Tally {
17238            first_choice: BTreeMap::from([('A', 1)]),
17239            borda: BTreeMap::new(),
17240            winner: 'A',
17241            rankings: 1,
17242            unanimous_initial: true,
17243            deliberated: false,
17244            changed_votes: 0,
17245            unanimous_final: true,
17246            tie_break: None,
17247            judges: 0,
17248            present: 0,
17249            quorum: 0,
17250            met_quorum: true,
17251            uncontested: Some("only candidate A produced a change".to_owned()),
17252        });
17253        state.reviews = vec![ReviewRound {
17254            round: 1,
17255            head: "deadbeef".to_owned(),
17256            verified_head: None,
17257            verified_at: None,
17258            reviews: Vec::new(),
17259            e2e: Vec::new(),
17260            fix: None,
17261            blocking: 0,
17262            answered: 0,
17263            expected: 0,
17264            clean: true,
17265            verify_retried: false,
17266            e2e_deferred: false,
17267            e2e_defer_reason: None,
17268            progressed: false,
17269            vote_split: false,
17270            reconsideration: Vec::new(),
17271            verdict: None,
17272        }];
17273        state.gate = vec![CommandOutcome {
17274            command: "test".to_owned(),
17275            code: Some(0),
17276            output_tail: String::new(),
17277            duration_ms: 0,
17278            resource_blocked: false,
17279        }];
17280        state.gate_ran = true;
17281        // Reached its conclusion already — e.g. `land` closing the PR without
17282        // merging it, which (like the honest `MergeMode::None` path) leaves
17283        // `status` at `Ready`. The recorded outcome is what actually marks
17284        // this node done.
17285        state.status = RunStatus::Ready;
17286        state.merge = Some(MergeOutcome {
17287            mode: MergeMode::Local,
17288            ok: false,
17289            detail: "already concluded".to_owned(),
17290            empty: false,
17291        });
17292
17293        let mut runner = Runner {
17294            state,
17295            roles: ResolvedRoles {
17296                implementers: Vec::new(),
17297                judges: Vec::new(),
17298                reviewers: Vec::new(),
17299                fixer: None,
17300                conductor: conductor(),
17301                implementer_roster: Vec::new(),
17302                judge_roster: Vec::new(),
17303                reviewer_roster: Vec::new(),
17304            },
17305            sem: Arc::new(Semaphore::new(1)),
17306            pause: Pause::new(),
17307            interrupt: Pause::new(),
17308        };
17309
17310        runner.merge().await.expect("merge");
17311
17312        assert_eq!(
17313            runner.state.status,
17314            RunStatus::Ready,
17315            "a concluded run's status must not change on reentry"
17316        );
17317        assert_eq!(
17318            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17319            Some("already concluded"),
17320            "merge must not run again once the node already recorded an outcome"
17321        );
17322    }
17323
17324    /// `gate` leaves `state.gate_ran` false both before it has ever run and
17325    /// when its last attempt was resource-blocked (the shared build cache
17326    /// could not be acquired or confirmed fresh in time - see
17327    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
17328    /// `Vec` this also leaves behind used to read as "nothing failed" and let
17329    /// a run merge a tree the gate never actually checked - exactly the case
17330    /// a contended cache produces on every retry until it clears. `merge`
17331    /// must refuse until `gate` has actually recorded an attempt.
17332    #[tokio::test]
17333    async fn merge_refuses_a_gate_that_has_not_actually_run() {
17334        let tmp = tempfile::tempdir().expect("tempdir");
17335        let repo = tmp.path().join("repo");
17336        std::fs::create_dir_all(&repo).unwrap();
17337        init_repo(&repo);
17338
17339        let mut config = Config::default();
17340        config.merge.mode = MergeMode::Local;
17341
17342        let mut state = RunState::new(
17343            repo.clone(),
17344            "main".to_owned(),
17345            "deadbeef".to_owned(),
17346            "task".to_owned(),
17347            config,
17348        );
17349        state.candidates = vec![Candidate {
17350            index: 0,
17351            label: 'A',
17352            agent: "alpha".to_owned(),
17353            branch: "does-not-exist".to_owned(),
17354            worktree: repo.clone(),
17355            summary: String::new(),
17356            stat: String::new(),
17357            files: 0,
17358            commits: 0,
17359            empty: false,
17360            failed: None,
17361            verified_noop: None,
17362            duration_ms: 0,
17363            folded: false,
17364        }];
17365        state.tally = Some(Tally {
17366            first_choice: BTreeMap::from([('A', 1)]),
17367            borda: BTreeMap::new(),
17368            winner: 'A',
17369            rankings: 1,
17370            unanimous_initial: true,
17371            deliberated: false,
17372            changed_votes: 0,
17373            unanimous_final: true,
17374            tie_break: None,
17375            judges: 0,
17376            present: 0,
17377            quorum: 0,
17378            met_quorum: true,
17379            uncontested: Some("only candidate A produced a change".to_owned()),
17380        });
17381        state.reviews = vec![ReviewRound {
17382            round: 1,
17383            head: "deadbeef".to_owned(),
17384            verified_head: None,
17385            verified_at: None,
17386            reviews: Vec::new(),
17387            e2e: Vec::new(),
17388            fix: None,
17389            blocking: 0,
17390            answered: 0,
17391            expected: 0,
17392            clean: true,
17393            verify_retried: false,
17394            e2e_deferred: false,
17395            e2e_defer_reason: None,
17396            progressed: false,
17397            vote_split: false,
17398            reconsideration: Vec::new(),
17399            verdict: None,
17400        }];
17401        // The point: `gate` has not recorded anything yet.
17402        state.gate = Vec::new();
17403        state.gate_ran = false;
17404        state.status = RunStatus::Gating;
17405
17406        let mut runner = Runner {
17407            state,
17408            roles: ResolvedRoles {
17409                implementers: Vec::new(),
17410                judges: Vec::new(),
17411                reviewers: Vec::new(),
17412                fixer: None,
17413                conductor: conductor(),
17414                implementer_roster: Vec::new(),
17415                judge_roster: Vec::new(),
17416                reviewer_roster: Vec::new(),
17417            },
17418            sem: Arc::new(Semaphore::new(1)),
17419            pause: Pause::new(),
17420            interrupt: Pause::new(),
17421        };
17422
17423        runner.merge().await.expect("merge");
17424
17425        assert!(
17426            runner.state.merge.is_none(),
17427            "an empty gate must never be read as a passing one: {:?}",
17428            runner.state.merge
17429        );
17430    }
17431
17432    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
17433    /// commands configured and `merge.mode` is `none` (a review-only run).
17434    /// `gate` must still record a real attempt — zero commands, vacuously
17435    /// passed — rather than leaving `state.gate` empty in a way `merge`
17436    /// cannot tell apart from "never ran"; otherwise the run reaches
17437    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
17438    #[tokio::test]
17439    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
17440        let tmp = tempfile::tempdir().expect("tempdir");
17441        let repo = tmp.path().join("repo");
17442        std::fs::create_dir_all(&repo).unwrap();
17443        init_repo(&repo);
17444
17445        // Default config: `verify.gate` empty, `merge.mode` is `none`.
17446        let config = Config::default();
17447
17448        let mut state = RunState::new(
17449            repo.clone(),
17450            "main".to_owned(),
17451            "deadbeef".to_owned(),
17452            "task".to_owned(),
17453            config,
17454        );
17455        state.candidates = vec![Candidate {
17456            index: 0,
17457            label: 'A',
17458            agent: "alpha".to_owned(),
17459            branch: "does-not-exist".to_owned(),
17460            worktree: repo.clone(),
17461            summary: String::new(),
17462            stat: String::new(),
17463            files: 0,
17464            commits: 0,
17465            empty: false,
17466            failed: None,
17467            verified_noop: None,
17468            duration_ms: 0,
17469            folded: false,
17470        }];
17471        state.tally = Some(Tally {
17472            first_choice: BTreeMap::from([('A', 1)]),
17473            borda: BTreeMap::new(),
17474            winner: 'A',
17475            rankings: 1,
17476            unanimous_initial: true,
17477            deliberated: false,
17478            changed_votes: 0,
17479            unanimous_final: true,
17480            tie_break: None,
17481            judges: 0,
17482            present: 0,
17483            quorum: 0,
17484            met_quorum: true,
17485            uncontested: Some("only candidate A produced a change".to_owned()),
17486        });
17487        state.reviews = vec![ReviewRound {
17488            round: 1,
17489            head: "deadbeef".to_owned(),
17490            verified_head: None,
17491            verified_at: None,
17492            reviews: Vec::new(),
17493            e2e: Vec::new(),
17494            fix: None,
17495            blocking: 0,
17496            answered: 0,
17497            expected: 0,
17498            clean: true,
17499            verify_retried: false,
17500            e2e_deferred: false,
17501            e2e_defer_reason: None,
17502            progressed: false,
17503            vote_split: false,
17504            reconsideration: Vec::new(),
17505            verdict: None,
17506        }];
17507
17508        let mut runner = Runner {
17509            state,
17510            roles: ResolvedRoles {
17511                implementers: Vec::new(),
17512                judges: Vec::new(),
17513                reviewers: Vec::new(),
17514                fixer: None,
17515                conductor: conductor(),
17516                implementer_roster: Vec::new(),
17517                judge_roster: Vec::new(),
17518                reviewer_roster: Vec::new(),
17519            },
17520            sem: Arc::new(Semaphore::new(1)),
17521            pause: Pause::new(),
17522            interrupt: Pause::new(),
17523        };
17524
17525        runner.gate().await.expect("gate");
17526        assert!(
17527            runner.state.gate_ran,
17528            "zero configured commands is still a real attempt, not an unrun gate"
17529        );
17530        assert!(runner.state.gate.is_empty());
17531        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17532        assert_ne!(
17533            runner.state.status,
17534            RunStatus::Blocked,
17535            "a gate with nothing to check must not read as failed"
17536        );
17537
17538        runner.merge().await.expect("merge");
17539        assert_eq!(
17540            runner.state.status,
17541            RunStatus::Ready,
17542            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17543        );
17544    }
17545
17546    /// `Config::cache_dir` is derived from `verify.e2e` as well as
17547    /// `verify.gate` (so the e2e leg and the final gate never build against
17548    /// different directories). With zero `verify.gate` commands but a
17549    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
17550    /// that lease before discovering it had nothing to run - so a repo with
17551    /// no gate commands could come back `resource_blocked` (and therefore
17552    /// still `gate_ran == false`) on nothing but an unrelated run holding the
17553    /// cache, exactly the contention this run's own zero commands could
17554    /// never have touched. `gate` must recognise there is nothing to check
17555    /// before it ever asks for the lease.
17556    #[tokio::test]
17557    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17558        crate::run::pin_test_home();
17559        let home = crate::run::home();
17560
17561        let tmp = tempfile::tempdir().expect("tempdir");
17562        let repo = tmp.path().join("repo");
17563        std::fs::create_dir_all(&repo).unwrap();
17564        init_repo(&repo);
17565        // Unique to this test, so holding its lease cannot collide with
17566        // another test sharing the same process-wide `home`.
17567        let cache_dir = tmp.path().join("target");
17568
17569        let mut config = Config::default();
17570        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17571        // `verify.gate` stays empty (the default). Bounded so a regression
17572        // that does start waiting fails the test in seconds, not hangs it.
17573        config.graph.timeout_verify = Some(2);
17574        config.graph.timeout_resource_wait = 2;
17575
17576        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17577        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17578            .expect("no io error acquiring directly")
17579        {
17580            crate::cache::AcquireOutcome::Acquired(g) => g,
17581            crate::cache::AcquireOutcome::Busy(b) => {
17582                panic!("expected the direct acquire to win the lease first: {b:?}")
17583            }
17584        };
17585
17586        let mut state = RunState::new(
17587            repo.clone(),
17588            "main".to_owned(),
17589            "deadbeef".to_owned(),
17590            "task".to_owned(),
17591            config,
17592        );
17593        state.candidates = vec![Candidate {
17594            index: 0,
17595            label: 'A',
17596            agent: "alpha".to_owned(),
17597            branch: "does-not-exist".to_owned(),
17598            worktree: repo.clone(),
17599            summary: String::new(),
17600            stat: String::new(),
17601            files: 0,
17602            commits: 0,
17603            empty: false,
17604            failed: None,
17605            verified_noop: None,
17606            duration_ms: 0,
17607            folded: false,
17608        }];
17609        state.tally = Some(Tally {
17610            first_choice: BTreeMap::from([('A', 1)]),
17611            borda: BTreeMap::new(),
17612            winner: 'A',
17613            rankings: 1,
17614            unanimous_initial: true,
17615            deliberated: false,
17616            changed_votes: 0,
17617            unanimous_final: true,
17618            tie_break: None,
17619            judges: 0,
17620            present: 0,
17621            quorum: 0,
17622            met_quorum: true,
17623            uncontested: Some("only candidate A produced a change".to_owned()),
17624        });
17625        state.reviews = vec![ReviewRound {
17626            round: 1,
17627            head: "deadbeef".to_owned(),
17628            verified_head: None,
17629            verified_at: None,
17630            reviews: Vec::new(),
17631            e2e: Vec::new(),
17632            fix: None,
17633            blocking: 0,
17634            answered: 0,
17635            expected: 0,
17636            clean: true,
17637            verify_retried: false,
17638            e2e_deferred: false,
17639            e2e_defer_reason: None,
17640            progressed: false,
17641            vote_split: false,
17642            reconsideration: Vec::new(),
17643            verdict: None,
17644        }];
17645
17646        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
17647        state.reviews[0].e2e = vec![CommandOutcome {
17648            command: state.config.verify.e2e[0].clone(),
17649            code: Some(0),
17650            output_tail: String::new(),
17651            duration_ms: 0,
17652            resource_blocked: false,
17653        }];
17654        let mut runner = Runner {
17655            state,
17656            roles: ResolvedRoles {
17657                implementers: Vec::new(),
17658                judges: Vec::new(),
17659                reviewers: Vec::new(),
17660                fixer: None,
17661                conductor: conductor(),
17662                implementer_roster: Vec::new(),
17663                judge_roster: Vec::new(),
17664                reviewer_roster: Vec::new(),
17665            },
17666            sem: Arc::new(Semaphore::new(1)),
17667            pause: Pause::new(),
17668            interrupt: Pause::new(),
17669        };
17670
17671        let started = std::time::Instant::now();
17672        runner.gate().await.expect("gate");
17673        assert!(
17674            started.elapsed() < Duration::from_secs(1),
17675            "a gate with nothing to run must never wait on a lease it never needed"
17676        );
17677        assert!(
17678            runner.state.gate_ran,
17679            "zero commands is still a real, immediate attempt"
17680        );
17681        assert!(runner.state.gate.is_empty());
17682        assert_ne!(
17683            runner.state.status,
17684            RunStatus::Blocked,
17685            "must not read as resource-blocked on a lease it never asked for"
17686        );
17687    }
17688
17689    /// A separate process carries a live worker's environment throughout all
17690    /// verification entry points; the parallel harness never changes its env.
17691    #[cfg(unix)]
17692    #[tokio::test]
17693    async fn verification_is_standalone_inside_a_live_worker() {
17694        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17695        if std::env::var_os(CHILD).is_none() {
17696            let home = tempfile::tempdir().unwrap();
17697            let generation = crate::worker::pin_current(home.path()).unwrap();
17698            let output = std::process::Command::new(std::env::current_exe().unwrap())
17699                .args([
17700                    "--exact",
17701                    "graph::tests::verification_is_standalone_inside_a_live_worker",
17702                    "--nocapture",
17703                ])
17704                .env(CHILD, "1")
17705                .env("MAGI_HOME", home.path())
17706                .env("MAGI_WORKER_HOME", home.path())
17707                .env("MAGI_WORKER_ID", "abcdef")
17708                .env("MAGI_WORKER_GENERATION", &generation.id)
17709                .env("MAGI_WORKER_EXE", &generation.executable)
17710                .env("MAGI_RUN", "verification-parent")
17711                .env("MAGI_NODE", "gate")
17712                .env("MAGI_SEAT", "worker-fixture")
17713                .env("MAGI_PROCESS_OWNER", "fixture-owner")
17714                .env("MAGI_NO_AUTOUPDATE", "0")
17715                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17716                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17717                .env(
17718                    "VERIFY_EXPECT_TOOLCHAIN",
17719                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17720                )
17721                .env(
17722                    "VERIFY_EXPECT_CACHE",
17723                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17724                )
17725                .output()
17726                .unwrap();
17727            assert!(
17728                output.status.success()
17729                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17730                "{}\n{}",
17731                String::from_utf8_lossy(&output.stdout),
17732                String::from_utf8_lossy(&output.stderr)
17733            );
17734            return;
17735        }
17736        let home = crate::run::home();
17737        let directory = home.join("workers/jobs/abcdef");
17738        std::fs::create_dir_all(&directory).unwrap();
17739        let job = crate::worker::Job {
17740            id: "abcdef".into(),
17741            sequence: 1,
17742            kind: "task".into(),
17743            key: "task:verification".into(),
17744            payload: serde_json::json!({}),
17745            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17746            version: Some(env!("CARGO_PKG_VERSION").into()),
17747            status: crate::worker::Status::Running,
17748            result: None,
17749            pid: Some(std::process::id()),
17750            created_at: 0,
17751            finished_at: None,
17752        };
17753        std::fs::write(
17754            directory.join("job.json"),
17755            serde_json::to_vec(&job).unwrap(),
17756        )
17757        .unwrap();
17758        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17759        let assert_worker = || {
17760            assert_eq!(
17761                crate::worker::current_job(&home).unwrap().unwrap().id,
17762                job.id
17763            );
17764            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17765            assert_eq!(
17766                crate::worker::helper_executable(&home).unwrap(),
17767                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17768            );
17769            assert!(
17770                crate::updater::Checker::new(&crate::config::Update {
17771                    mode: crate::config::UpdateMode::Notify,
17772                    interval: None,
17773                    auto_restart: false,
17774                })
17775                .is_none(),
17776                "the driver remains pinned, even while tests are standalone"
17777            );
17778        };
17779        assert_worker();
17780        let repo = home.join("repo");
17781        std::fs::create_dir_all(&repo).unwrap();
17782        init_repo(&repo);
17783        // Keep storage, seat/control context, containment and tool lookup.
17784        // No inherited values (or credentials) are printed on success.
17785        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17786            test -z "${MAGI_WORKER_GENERATION+x}" &&
17787            test -z "${MAGI_WORKER_HOME+x}" &&
17788            test -z "${MAGI_WORKER_EXE+x}" &&
17789            test "$MAGI_HOME/repo" -ef . &&
17790            test "$MAGI_RUN" = verification-parent &&
17791            test "$MAGI_NODE" = gate &&
17792            test "$MAGI_SEAT" = worker-fixture &&
17793            test -n "$MAGI_PROCESS_OWNER" &&
17794            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17795            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17796            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17797            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17798        // Negative control: an ordinary child inherits the contamination.
17799        assert!(
17800            !std::process::Command::new("sh")
17801                .args(["-c", probe])
17802                .current_dir(&repo)
17803                .status()
17804                .unwrap()
17805                .success()
17806        );
17807        let unrelated_home = tempfile::tempdir().unwrap();
17808        assert!(
17809            crate::worker::current_job(unrelated_home.path()).is_err(),
17810            "an inherited worker id cannot be resolved in a unit fixture's home"
17811        );
17812        let mut runner = runner_at(RunStatus::Gating);
17813        runner.state.repo = repo.clone();
17814        runner.state.config.disk.min_free_bytes = 0;
17815        runner.state.config.coordination = crate::config::test_coordination();
17816        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17817        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17818        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17819        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17820        let winner = Candidate {
17821            index: 0,
17822            label: 'A',
17823            agent: "fixture".into(),
17824            branch: "main".into(),
17825            worktree: repo.clone(),
17826            summary: String::new(),
17827            stat: String::new(),
17828            files: 0,
17829            commits: 0,
17830            empty: false,
17831            failed: None,
17832            verified_noop: None,
17833            duration_ms: 0,
17834            folded: false,
17835        };
17836        let (e2e, retried, pids) = run_e2e_with_retry(
17837            &mut runner.state,
17838            &["sh".into(), "-c".into()],
17839            &[format!("{probe} && printf initial")],
17840            &repo,
17841            Duration::from_secs(10),
17842            "initial",
17843        )
17844        .await;
17845        assert!(!retried && pids.is_empty());
17846        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17847        assert_eq!(e2e[0].output_tail, "initial");
17848        let gate = runner.run_gate(&winner).await.unwrap();
17849        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17850        // An interrupted repair must recheck e2e from saved state. Landing
17851        // reuses that state and must run both pre-gate and gate again.
17852        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17853            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17854            passed: false,
17855            outcomes: Vec::new(),
17856        });
17857        runner.state.save().unwrap();
17858        let mut resumed = RunState::load(&runner.state.id).unwrap();
17859        let wait = crate::repo_lock::WaitPolicy::default();
17860        assert!(matches!(
17861            check_gate_repair_e2e(&mut resumed, &winner, &wait)
17862                .await
17863                .unwrap(),
17864            GateFix::Retry
17865        ));
17866        let saved = RunState::load(&resumed.id).unwrap();
17867        let repair = saved.gate_repair_e2e.unwrap();
17868        assert!(repair.passed);
17869        assert_eq!(repair.outcomes.len(), 1);
17870        assert!(repair.outcomes[0].ok());
17871        let landing = run_landing_gate(&mut resumed, &winner, &wait)
17872            .await
17873            .unwrap();
17874        assert_eq!(landing.len(), 1);
17875        assert!(landing[0].ok(), "{landing:?}");
17876        assert_eq!(
17877            std::fs::read_to_string(repo.join("phases")).unwrap(),
17878            "pre\ngate\ne2e\npre\ngate\n"
17879        );
17880        assert_worker();
17881    }
17882
17883    #[cfg(unix)]
17884    #[tokio::test]
17885    async fn verification_keeps_the_panic_and_the_full_output() {
17886        crate::run::pin_test_home();
17887        let tmp = tempfile::tempdir().unwrap();
17888        let mut config = Config::default();
17889        config.disk.min_free_bytes = 0;
17890        let mut state = RunState::new(
17891            tmp.path().to_owned(),
17892            "main".into(),
17893            "base".into(),
17894            "verification evidence".into(),
17895            config,
17896        );
17897        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
17898                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
17899                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
17900        let mut names = Vec::new();
17901        for _ in 0..2 {
17902            let (outcomes, _) = run_commands(
17903                &mut state,
17904                "gate",
17905                "gate",
17906                0,
17907                &["sh".into(), "-c".into()],
17908                &[script.into()],
17909                tmp.path(),
17910                Duration::from_secs(60),
17911            )
17912            .await;
17913            let tail = &outcomes[0].output_tail;
17914            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
17915            assert!(tail.contains("boom assertion"));
17916            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
17917            let line = tail.lines().next().unwrap();
17918            names.push(line.to_owned());
17919        }
17920        assert_ne!(names[0], names[1], "the second run must not overwrite");
17921        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
17922        assert!(full.len() > 500_000 && full.contains("boom assertion"));
17923        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
17924
17925        // Storage failures are evidence too; preserve the command's verdict.
17926        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
17927        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
17928        let (outcomes, _) = run_commands(
17929            &mut state,
17930            "gate",
17931            "gate",
17932            0,
17933            &["sh".into(), "-c".into()],
17934            &["echo failure-output; exit 7".into()],
17935            tmp.path(),
17936            Duration::from_secs(10),
17937        )
17938        .await;
17939        assert_eq!(outcomes[0].code, Some(7));
17940        assert!(state.events.iter().any(|e| {
17941            e.node == "gate" && e.message.contains("could not save the full gate output:")
17942        }));
17943    }
17944
17945    #[cfg(unix)]
17946    #[tokio::test]
17947    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
17948        crate::run::pin_test_home();
17949        let tmp = tempfile::tempdir().unwrap();
17950        let mut config = Config::default();
17951        config.disk.min_free_bytes = 0;
17952        let mut state = RunState::new(
17953            tmp.path().to_owned(),
17954            "main".into(),
17955            "base".into(),
17956            "verification evidence".into(),
17957            config,
17958        );
17959        let (outcomes, pids) = run_commands(
17960            &mut state,
17961            "gate",
17962            "gate",
17963            1,
17964            &["sh".into(), "-c".into()],
17965            &[
17966                "printf before-timeout; exec sleep 30".into(),
17967                "kill -TERM $$".into(),
17968                "exit 7".into(),
17969                "printf success".into(),
17970            ],
17971            tmp.path(),
17972            Duration::from_secs(1),
17973        )
17974        .await;
17975        wait_for_timed_out_children_to_die(&pids).await;
17976        assert_eq!(
17977            pids.len(),
17978            1,
17979            "unleased timeout retains its death-check PID"
17980        );
17981        assert_eq!(outcomes.len(), 4);
17982        assert_eq!(outcomes[0].code, None);
17983        assert!(outcomes[0].output_tail.contains("before-timeout"));
17984        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
17985        assert_eq!(outcomes[1].code, None);
17986        assert!(outcomes[1].output_tail.contains("command terminated:"));
17987        assert!(!outcomes[1].output_tail.contains("timed out"));
17988        assert_eq!(outcomes[2].code, Some(7));
17989        assert!(!outcomes[2].output_tail.contains("timed out"));
17990        assert_eq!(outcomes[3].code, Some(0));
17991        assert_eq!(outcomes[3].output_tail, "success");
17992        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
17993    }
17994
17995    /// The addendum's second gap: a `verify.gate` command running for real
17996    /// wall-clock time had nothing at all to show for it in `active` before
17997    /// `run_commands` learned to record it — a run could sit in `Gating` for
17998    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
17999    /// what was actually happening. Proven with a genuinely still-running
18000    /// command, not just a before/after check on the final state: a poller
18001    /// task reads the same `run.json` `gate()` is writing, the same way the
18002    /// phone or `magi show` would, while the shell command is still blocked
18003    /// on its own release marker.
18004    #[tokio::test]
18005    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18006        crate::run::pin_test_home();
18007
18008        let tmp = tempfile::tempdir().expect("tempdir");
18009        let repo = tmp.path().join("repo");
18010        std::fs::create_dir_all(&repo).unwrap();
18011        init_repo(&repo);
18012
18013        let mut config = Config::default();
18014        config.verify.gate = vec![
18015            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18016             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18017                .to_owned(),
18018        ];
18019
18020        let mut state = RunState::new(
18021            repo.clone(),
18022            "main".to_owned(),
18023            "deadbeef".to_owned(),
18024            "task".to_owned(),
18025            config,
18026        );
18027        let run_id = state.id.clone();
18028        state.candidates = vec![Candidate {
18029            index: 0,
18030            label: 'A',
18031            agent: "alpha".to_owned(),
18032            branch: "does-not-exist".to_owned(),
18033            worktree: repo.clone(),
18034            summary: String::new(),
18035            stat: String::new(),
18036            files: 0,
18037            commits: 0,
18038            empty: false,
18039            failed: None,
18040            verified_noop: None,
18041            duration_ms: 0,
18042            folded: false,
18043        }];
18044        state.tally = Some(Tally {
18045            first_choice: BTreeMap::from([('A', 1)]),
18046            borda: BTreeMap::new(),
18047            winner: 'A',
18048            rankings: 1,
18049            unanimous_initial: true,
18050            deliberated: false,
18051            changed_votes: 0,
18052            unanimous_final: true,
18053            tie_break: None,
18054            judges: 0,
18055            present: 0,
18056            quorum: 0,
18057            met_quorum: true,
18058            uncontested: Some("only candidate A produced a change".to_owned()),
18059        });
18060        state.reviews = vec![ReviewRound {
18061            round: 1,
18062            head: "deadbeef".to_owned(),
18063            verified_head: None,
18064            verified_at: None,
18065            reviews: Vec::new(),
18066            e2e: Vec::new(),
18067            fix: None,
18068            blocking: 0,
18069            answered: 0,
18070            expected: 0,
18071            clean: true,
18072            verify_retried: false,
18073            e2e_deferred: false,
18074            e2e_defer_reason: None,
18075            progressed: false,
18076            vote_split: false,
18077            reconsideration: Vec::new(),
18078            verdict: None,
18079        }];
18080
18081        let mut runner = Runner {
18082            state,
18083            roles: ResolvedRoles {
18084                implementers: Vec::new(),
18085                judges: Vec::new(),
18086                reviewers: Vec::new(),
18087                fixer: None,
18088                conductor: conductor(),
18089                implementer_roster: Vec::new(),
18090                judge_roster: Vec::new(),
18091                reviewer_roster: Vec::new(),
18092            },
18093            sem: Arc::new(Semaphore::new(1)),
18094            pause: Pause::new(),
18095            interrupt: Pause::new(),
18096        };
18097
18098        let started_marker = repo.join("started.marker");
18099        let release_marker = repo.join("release.marker");
18100        let poller = tokio::spawn(async move {
18101            // Bounded so a regression that never records the task entry
18102            // fails this test in seconds instead of hanging the suite —
18103            // the same shape `a_park_requested_while_a_seat_is_mid_call_
18104            // does_not_cut_it_short` uses for the same reason.
18105            for _ in 0..100 {
18106                if started_marker.exists()
18107                    && let Ok(s) = crate::run::RunState::load(&run_id)
18108                    && let Some(a) = s.active.get("gate")
18109                {
18110                    std::fs::write(&release_marker, b"go").expect("release marker");
18111                    return Some(a.clone());
18112                }
18113                tokio::time::sleep(Duration::from_millis(50)).await;
18114            }
18115            None
18116        });
18117
18118        runner.gate().await.expect("gate");
18119        let captured = poller.await.expect("poller task");
18120        let captured = captured.expect(
18121            "the poller never saw a `gate` task entry in run.json while the command was \
18122             still blocked on its own release marker",
18123        );
18124
18125        assert_eq!(captured.task.as_deref(), Some("gate"));
18126        assert_eq!(captured.node, "gate");
18127        assert_eq!(captured.index, Some(1));
18128        assert_eq!(captured.total, Some(1));
18129        assert!(
18130            captured
18131                .command
18132                .as_deref()
18133                .is_some_and(|c| c.contains("started.marker")),
18134            "{captured:?}"
18135        );
18136
18137        assert!(
18138            runner.state.active.is_empty(),
18139            "the entry must be cleared once the command actually finished: {:?}",
18140            runner.state.active
18141        );
18142        assert!(runner.state.gate_ran);
18143        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18144    }
18145
18146    /// The hand-off over a blocking finding a reviewer rejected on leaves a
18147    /// record for `land`; one with only a Minor, or no reject, leaves none.
18148    #[tokio::test]
18149    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18150        use crate::verdict::{Finding, ReviewVote, Severity};
18151        crate::run::pin_test_home();
18152        let tmp = tempfile::tempdir().expect("tempdir");
18153        let repo = tmp.path().join("repo");
18154        std::fs::create_dir_all(&repo).unwrap();
18155        init_repo(&repo);
18156
18157        for (severity, vote, expect) in [
18158            (Severity::Major, ReviewVote::Reject, true),
18159            (Severity::Minor, ReviewVote::Reject, false),
18160            (Severity::Major, ReviewVote::Approve, false),
18161        ] {
18162            let mut round = review_round(false, 1, 1, 1, false, true);
18163            round.reviews = vec![ReviewRecord {
18164                reviewer: 1,
18165                agent: "alpha".to_owned(),
18166                summary: String::new(),
18167                findings: vec![Finding {
18168                    id: "R1-1-1".to_owned(),
18169                    severity,
18170                    file: None,
18171                    line: None,
18172                    title: "t".to_owned(),
18173                    detail: String::new(),
18174                }],
18175                vote: Some(vote),
18176                failed: None,
18177                duration_ms: 0,
18178                attempts: 0,
18179            }];
18180            let mut state = RunState::new(
18181                repo.clone(),
18182                "main".to_owned(),
18183                "deadbeef".to_owned(),
18184                "task".to_owned(),
18185                Config::default(),
18186            );
18187            state.reviews = vec![round];
18188            let mut runner = Runner {
18189                state,
18190                roles: ResolvedRoles {
18191                    implementers: Vec::new(),
18192                    judges: Vec::new(),
18193                    reviewers: Vec::new(),
18194                    fixer: None,
18195                    conductor: conductor(),
18196                    implementer_roster: Vec::new(),
18197                    judge_roster: Vec::new(),
18198                    reviewer_roster: Vec::new(),
18199                },
18200                sem: Arc::new(Semaphore::new(1)),
18201                pause: Pause::new(),
18202                interrupt: Pause::new(),
18203            };
18204            let shell = runner.state.config.shell();
18205            runner
18206                .stop_reviewing("round budget spent", &shell, &repo)
18207                .await
18208                .expect("stop_reviewing");
18209            assert_eq!(runner.state.status, RunStatus::Gating);
18210            assert_eq!(
18211                runner.state.contested_handoff.is_some(),
18212                expect,
18213                "{severity:?} + {vote:?}"
18214            );
18215        }
18216    }
18217
18218    /// The shape the incident this whole fix responds to actually had: the
18219    /// round budget spent, the last round's own e2e blocked on the shared
18220    /// build cache (held here by a live pid — this test process — exactly
18221    /// `cache`'s own unit tests' pattern for "another owner, still alive"
18222    /// without forking a process). `stop_reviewing` must retry it — not
18223    /// silently leave the round looking untouched (the catch-up-only half of
18224    /// the bug), and not read the contention as a red `e2e` and block the
18225    /// run on it (the other half). Called directly, the same way
18226    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
18227    /// above exercises `gate`, so this never needs a real cargo build to
18228    /// reach: the lease is never released, so `with_cache_lease` never gets
18229    /// past acquiring it into anything that would need a real workspace.
18230    #[tokio::test]
18231    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18232        crate::run::pin_test_home();
18233        let home = crate::run::home();
18234
18235        let tmp = tempfile::tempdir().expect("tempdir");
18236        let repo = tmp.path().join("repo");
18237        std::fs::create_dir_all(&repo).unwrap();
18238        init_repo(&repo);
18239        let head = crate::git::rev_parse(&repo, "HEAD")
18240            .await
18241            .expect("rev-parse");
18242        // Unique to this test, so holding its lease cannot collide with
18243        // another test sharing the same process-wide `home`.
18244        let cache_dir = tmp.path().join("target");
18245
18246        let mut config = Config::default();
18247        config.verify.e2e = vec![format!(
18248            "CARGO_TARGET_DIR='{}' test -f README.md",
18249            cache_dir.display()
18250        )];
18251        config.graph.review_rounds = 1;
18252        // Bounded so a regression that does start waiting fails the test in
18253        // seconds, not hangs it.
18254        config.graph.timeout_verify = Some(2);
18255        config.graph.timeout_resource_wait = 2;
18256
18257        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18258        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18259            .expect("no io error acquiring directly")
18260        {
18261            crate::cache::AcquireOutcome::Acquired(g) => g,
18262            crate::cache::AcquireOutcome::Busy(b) => {
18263                panic!("expected the direct acquire to win the lease first: {b:?}")
18264            }
18265        };
18266
18267        let mut state = RunState::new(
18268            repo.clone(),
18269            "main".to_owned(),
18270            head.clone(),
18271            "task".to_owned(),
18272            config,
18273        );
18274        state.candidates = vec![Candidate {
18275            index: 0,
18276            label: 'A',
18277            agent: "alpha".to_owned(),
18278            branch: "does-not-exist".to_owned(),
18279            worktree: repo.clone(),
18280            summary: String::new(),
18281            stat: String::new(),
18282            files: 0,
18283            commits: 0,
18284            empty: false,
18285            failed: None,
18286            verified_noop: None,
18287            duration_ms: 0,
18288            folded: false,
18289        }];
18290        state.tally = Some(Tally {
18291            first_choice: BTreeMap::from([('A', 1)]),
18292            borda: BTreeMap::new(),
18293            winner: 'A',
18294            rankings: 1,
18295            unanimous_initial: true,
18296            deliberated: false,
18297            changed_votes: 0,
18298            unanimous_final: true,
18299            tie_break: None,
18300            judges: 0,
18301            present: 0,
18302            quorum: 0,
18303            met_quorum: true,
18304            uncontested: Some("only candidate A produced a change".to_owned()),
18305        });
18306        // The round budget's last round, deferred: `needs_catchup_run`'s
18307        // other trigger. `stop_reviewing`'s retry machinery must treat this
18308        // exactly like a resource-blocked attempt once it actually runs.
18309        state.reviews = vec![ReviewRound {
18310            round: 1,
18311            head: head.clone(),
18312            verified_head: None,
18313            verified_at: None,
18314            reviews: Vec::new(),
18315            e2e: Vec::new(),
18316            fix: None,
18317            blocking: 1,
18318            answered: 1,
18319            expected: 1,
18320            clean: false,
18321            verify_retried: false,
18322            e2e_deferred: true,
18323            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18324            progressed: false,
18325            vote_split: false,
18326            reconsideration: Vec::new(),
18327            verdict: None,
18328        }];
18329
18330        let mut runner = Runner {
18331            state,
18332            roles: ResolvedRoles {
18333                implementers: Vec::new(),
18334                judges: Vec::new(),
18335                reviewers: Vec::new(),
18336                fixer: None,
18337                conductor: conductor(),
18338                implementer_roster: Vec::new(),
18339                judge_roster: Vec::new(),
18340                reviewer_roster: Vec::new(),
18341            },
18342            sem: Arc::new(Semaphore::new(1)),
18343            pause: Pause::new(),
18344            interrupt: Pause::new(),
18345        };
18346
18347        let shell = runner.state.config.shell();
18348        runner
18349            .stop_reviewing("round budget spent", &shell, &repo)
18350            .await
18351            .expect("stop_reviewing");
18352
18353        let last = runner.state.reviews.last().expect("round record");
18354        assert_eq!(
18355            last.e2e_status(),
18356            E2eStatus::ResourceBlocked,
18357            "the shared cache is still held; the attempt must read as blocked, not deferred or \
18358             failed: {last:?}"
18359        );
18360        assert_eq!(
18361            last.verified_head.as_deref(),
18362            Some(head.as_str()),
18363            "which commit this attempt targeted is known even though nothing finished checking \
18364             it"
18365        );
18366        let first_attempt_at = last
18367            .verified_at
18368            .expect("when this attempt ran is known too");
18369        assert_ne!(
18370            runner.state.status,
18371            RunStatus::Blocked,
18372            "contention is evidence about the machine, not the patch — it must not settle the \
18373             run as blocked: {:?}",
18374            runner.state.status
18375        );
18376        assert!(
18377            !runner
18378                .state
18379                .events
18380                .iter()
18381                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18382            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18383            runner.state.events
18384        );
18385
18386        // The cache is still held: a later reentry must retry the same
18387        // round's verification again — not leave it looking exactly as
18388        // untouched as the first blocked attempt, which is indistinguishable
18389        // from never having tried again at all.
18390        runner
18391            .stop_reviewing("round budget spent", &shell, &repo)
18392            .await
18393            .expect("stop_reviewing retry");
18394        assert_eq!(
18395            runner.state.reviews.len(),
18396            1,
18397            "no new round was started: {:?}",
18398            runner.state.reviews
18399        );
18400        let last = runner.state.reviews.last().expect("round record");
18401        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18402        assert!(
18403            last.verified_at.expect("still known") > first_attempt_at,
18404            "a second reentry must be a fresh attempt, not a stale copy of the first"
18405        );
18406        assert_ne!(runner.state.status, RunStatus::Blocked);
18407
18408        held.release();
18409    }
18410
18411    #[cfg(unix)]
18412    #[tokio::test]
18413    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
18414        ask_test_home();
18415        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
18416            let tmp = tempfile::tempdir().unwrap();
18417            let repo = tmp.path().join("repo");
18418            std::fs::create_dir(&repo).unwrap();
18419            init_repo(&repo);
18420            let run_git = |args: &[&str]| {
18421                let out = std::process::Command::new("git")
18422                    .args(args)
18423                    .current_dir(&repo)
18424                    .output()
18425                    .unwrap();
18426                assert!(
18427                    out.status.success(),
18428                    "{args:?}: {}",
18429                    String::from_utf8_lossy(&out.stderr)
18430                );
18431            };
18432            let origin = tmp.path().join("origin.git");
18433            run_git(&["init", "--bare", origin.to_str().unwrap()]);
18434            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
18435            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
18436            run_git(&["add", "spec.rs"]);
18437            run_git(&["commit", "-m", "initial spec"]);
18438            run_git(&["push", "origin", "main"]);
18439            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
18440            run_git(&["checkout", "-b", "fix/review"]);
18441            std::fs::write(
18442                repo.join("check.rs"),
18443                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
18444            )
18445            .unwrap();
18446            run_git(&["add", "check.rs"]);
18447            run_git(&["commit", "-m", "use spec"]);
18448            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
18449            run_git(&["checkout", "main"]);
18450            std::fs::write(
18451                repo.join("spec.rs"),
18452                "struct AgentSpec { max_turn_secs: u64 }\n",
18453            )
18454            .unwrap();
18455            run_git(&["add", "spec.rs"]);
18456            run_git(&["commit", "-m", "extend spec"]);
18457            run_git(&["push", "origin", "main"]);
18458            run_git(&["checkout", "fix/review"]);
18459
18460            let mut runner = runner_at(RunStatus::Reviewing);
18461            runner.state.repo = repo.clone();
18462            runner.state.base_commit = base;
18463            runner.state.pr_head = Some("fix/review".into());
18464            runner.state.config.graph.review_rounds = max_rounds;
18465            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
18466            runner.state.config.graph.sessions = false;
18467            runner.state.config.disk.min_free_bytes = 0;
18468            runner.state.config.verify.e2e =
18469                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
18470            runner.state.candidates = vec![Candidate {
18471                index: 0,
18472                label: 'A',
18473                agent: "fixture".into(),
18474                branch: "fix/review".into(),
18475                worktree: repo.clone(),
18476                summary: String::new(),
18477                stat: String::new(),
18478                files: 1,
18479                commits: 1,
18480                empty: false,
18481                failed: None,
18482                verified_noop: None,
18483                duration_ms: 0,
18484                folded: false,
18485            }];
18486            runner.state.tally = Some(Tally {
18487                first_choice: BTreeMap::new(),
18488                borda: BTreeMap::new(),
18489                winner: 'A',
18490                rankings: 0,
18491                unanimous_initial: true,
18492                deliberated: false,
18493                changed_votes: 0,
18494                unanimous_final: true,
18495                tie_break: None,
18496                judges: 0,
18497                present: 0,
18498                quorum: 0,
18499                met_quorum: true,
18500                uncontested: Some("review-only".into()),
18501            });
18502            let mut last = review_round(false, 0, 1, 1, false, false);
18503            last.head = reviewed.clone();
18504            last.e2e_deferred = true;
18505            last.e2e[0].resource_blocked = true;
18506            last.e2e[0].output_tail = "waiting for the shared build cache".into();
18507            if max_rounds == 1 {
18508                last.e2e.clear();
18509            }
18510            runner.state.reviews = vec![last];
18511            let calls = tmp.path().join("fix-calls");
18512            let mut fixer = spec("repair");
18513            fixer
18514                .env
18515                .insert("FIX_CALLS".into(), calls.display().to_string());
18516            fixer.command = vec!["sh".into(), "-c".into(), r#"
18517                echo fix >> "$FIX_CALLS"
18518                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18519                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18520                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18521                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18522            "#.into()];
18523            let mut reviewer = spec("reviewer");
18524            reviewer.command = vec![
18525                "sh".into(),
18526                "-c".into(),
18527                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18528                    .into(),
18529            ];
18530            runner.roles.fixer = Some(vec![fixer]);
18531            runner.roles.reviewers = vec![reviewer];
18532
18533            runner.sync_to_base().await.unwrap();
18534            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18535            assert_ne!(rebased, reviewed);
18536            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18537            if reentry {
18538                // Simulate a crash after catch-up evidence was saved but before
18539                // dispatching the fixer. No dedicated pending-repair field exists.
18540                let shell = runner.state.config.shell();
18541                runner
18542                    .stop_unresolved_review("deferred verification", &shell, &repo)
18543                    .await
18544                    .unwrap();
18545                assert!(runner.catchup_repairable());
18546                assert!(!calls.exists());
18547                runner.state = RunState::load(&runner.state.id).unwrap();
18548                assert!(runner.catchup_repairable());
18549            }
18550            runner.review_loop().await.unwrap();
18551            let first = &runner.state.reviews[0];
18552            assert_eq!(first.head, reviewed);
18553            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18554            assert!(
18555                first.verify_retried,
18556                "a reproduced build failure gets exactly one retry: {:?}",
18557                first.e2e
18558            );
18559            assert!(first.e2e[0].output_tail.contains("E0063"));
18560            assert!(!runner.catchup_repairable());
18561            if max_rounds == 2 {
18562                assert_eq!(
18563                    runner.state.status,
18564                    RunStatus::Gating,
18565                    "{}",
18566                    crate::report::run(&runner.state)
18567                );
18568                assert_eq!(runner.state.reviews.len(), 2);
18569                assert!(first.fix.as_ref().unwrap().committed);
18570                assert!(runner.state.reviews[1].clean);
18571                assert_eq!(
18572                    runner
18573                        .state
18574                        .reviews
18575                        .iter()
18576                        .filter(|r| r.fix.is_some())
18577                        .count(),
18578                    1
18579                );
18580                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18581                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18582            } else {
18583                assert_eq!(runner.state.status, RunStatus::Blocked);
18584                assert!(first.fix.is_none());
18585                assert!(!calls.exists());
18586                let recovery = crate::recovery::assess(
18587                    "prior",
18588                    Some(&runner.state),
18589                    &runner.landing_base(),
18590                    crate::recovery::Relation::Diverged,
18591                );
18592                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18593                let reasons = recovery.reasons.join("\n");
18594                assert!(
18595                    reasons.contains(&reviewed) && reasons.contains(&rebased),
18596                    "{reasons}"
18597                );
18598                assert!(
18599                    reasons.contains("base sync to")
18600                        && reasons.contains("additional review is required"),
18601                    "{reasons}"
18602                );
18603            }
18604        }
18605    }
18606
18607    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
18608    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
18609    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
18610    /// own top-of-function fast path (`review_conclusion`) correctly reads
18611    /// this shape as `None` rather than guessing `Blocked`, and the loop's
18612    /// own `for` range is empty once the round budget is spent, so
18613    /// `review_loop` must retry the check itself rather than silently doing
18614    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
18615    /// above exercises directly, but through `review_loop`'s own entry point
18616    /// this time, proving the wiring between the two rather than just the
18617    /// retry logic in isolation.
18618    #[tokio::test]
18619    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18620        crate::run::pin_test_home();
18621        let home = crate::run::home();
18622
18623        let tmp = tempfile::tempdir().expect("tempdir");
18624        let repo = tmp.path().join("repo");
18625        std::fs::create_dir_all(&repo).unwrap();
18626        init_repo(&repo);
18627        let head = crate::git::rev_parse(&repo, "HEAD")
18628            .await
18629            .expect("rev-parse");
18630        let cache_dir = tmp.path().join("target");
18631
18632        let mut config = Config::default();
18633        config.verify.e2e = vec![format!(
18634            "CARGO_TARGET_DIR='{}' test -f README.md",
18635            cache_dir.display()
18636        )];
18637        config.graph.review_rounds = 1;
18638        config.graph.timeout_verify = Some(2);
18639        config.graph.timeout_resource_wait = 2;
18640
18641        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18642        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18643            .expect("no io error acquiring directly")
18644        {
18645            crate::cache::AcquireOutcome::Acquired(g) => g,
18646            crate::cache::AcquireOutcome::Busy(b) => {
18647                panic!("expected the direct acquire to win the lease first: {b:?}")
18648            }
18649        };
18650
18651        let mut state = RunState::new(
18652            repo.clone(),
18653            "main".to_owned(),
18654            head.clone(),
18655            "task".to_owned(),
18656            config,
18657        );
18658        state.candidates = vec![Candidate {
18659            index: 0,
18660            label: 'A',
18661            agent: "alpha".to_owned(),
18662            branch: "does-not-exist".to_owned(),
18663            worktree: repo.clone(),
18664            summary: String::new(),
18665            stat: String::new(),
18666            files: 0,
18667            commits: 0,
18668            empty: false,
18669            failed: None,
18670            verified_noop: None,
18671            duration_ms: 0,
18672            folded: false,
18673        }];
18674        state.tally = Some(Tally {
18675            first_choice: BTreeMap::from([('A', 1)]),
18676            borda: BTreeMap::new(),
18677            winner: 'A',
18678            rankings: 1,
18679            unanimous_initial: true,
18680            deliberated: false,
18681            changed_votes: 0,
18682            unanimous_final: true,
18683            tie_break: None,
18684            judges: 0,
18685            present: 0,
18686            quorum: 0,
18687            met_quorum: true,
18688            uncontested: Some("only candidate A produced a change".to_owned()),
18689        });
18690        // The exact shape a prior process's `stop_reviewing` would have left
18691        // on disk: the round budget's last round, a real attempt already
18692        // made and already resource-blocked.
18693        state.reviews = vec![ReviewRound {
18694            round: 1,
18695            head: head.clone(),
18696            verified_head: Some(head.clone()),
18697            verified_at: Some(jiff::Timestamp::now()),
18698            reviews: Vec::new(),
18699            e2e: vec![CommandOutcome {
18700                command: format!(
18701                    "CARGO_TARGET_DIR='{}' test -f README.md",
18702                    cache_dir.display()
18703                ),
18704                code: None,
18705                output_tail: "waiting for the shared build cache".to_owned(),
18706                duration_ms: 0,
18707                resource_blocked: true,
18708            }],
18709            fix: None,
18710            blocking: 1,
18711            answered: 1,
18712            expected: 1,
18713            clean: false,
18714            verify_retried: false,
18715            e2e_deferred: false,
18716            e2e_defer_reason: None,
18717            progressed: false,
18718            vote_split: false,
18719            reconsideration: Vec::new(),
18720            verdict: None,
18721        }];
18722
18723        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18724        let mut runner = Runner {
18725            state,
18726            roles: ResolvedRoles {
18727                implementers: Vec::new(),
18728                judges: Vec::new(),
18729                reviewers: Vec::new(),
18730                fixer: None,
18731                conductor: conductor(),
18732                implementer_roster: Vec::new(),
18733                judge_roster: Vec::new(),
18734                reviewer_roster: Vec::new(),
18735            },
18736            sem: Arc::new(Semaphore::new(1)),
18737            pause: Pause::new(),
18738            interrupt: Pause::new(),
18739        };
18740
18741        // The lease is still held throughout, so this reentry's own retry is
18742        // also contended — proving `review_loop` actually tried again (not
18743        // that it happened to succeed) is what the timestamp comparison
18744        // below is for.
18745        runner.review_loop().await.expect("review_loop");
18746
18747        assert_eq!(
18748            runner.state.reviews.len(),
18749            1,
18750            "no new round was started on top of the unresolved one: {:?}",
18751            runner.state.reviews
18752        );
18753        let last = &runner.state.reviews[0];
18754        assert_eq!(
18755            last.e2e_status(),
18756            E2eStatus::ResourceBlocked,
18757            "still contended: {last:?}"
18758        );
18759        assert!(
18760            last.verified_at.expect("still known") > first_attempt_at,
18761            "review_loop must have actually retried the check, not left it exactly as found"
18762        );
18763        assert_ne!(
18764            runner.state.status,
18765            RunStatus::Blocked,
18766            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18767            runner.state.status
18768        );
18769
18770        held.release();
18771    }
18772
18773    #[tokio::test]
18774    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18775        crate::run::pin_test_home();
18776        for status in [RunStatus::Blocked, RunStatus::Gating] {
18777            let mut runner = runner_at(status);
18778            runner.state.config.graph.land = true;
18779            runner.state.gate_ran = true;
18780            runner.state.merge = Some(MergeOutcome {
18781                mode: MergeMode::Pr,
18782                ok: false,
18783                detail: "the fixer produced no commit".into(),
18784                empty: false,
18785            });
18786            let url = "https://example.invalid/o/r/pull/1";
18787            runner.state.pr = Some(crate::run::PrRecord {
18788                url: url.into(),
18789                number: 1,
18790                state: "open".into(),
18791                checks: "red".into(),
18792                round: 1,
18793                rounds: 1,
18794                red_at_merge: Vec::new(),
18795            });
18796            runner.state.save().unwrap();
18797            // The nonexistent repository makes observation fail before a forge
18798            // command can run. We still must reach land, rather than silently
18799            // return from the graph with its old gate and merge records.
18800            runner.execute_graph().await.unwrap();
18801            assert_eq!(runner.state.status, RunStatus::Blocked);
18802            assert!(runner.state.land_sync_pending);
18803            assert!(runner.state.events.iter().any(|event|
18804                event.node == "land" && event.message == format!("watching {url}")));
18805            assert_eq!(
18806                runner.state.merge.as_ref().unwrap().detail,
18807                "the fixer produced no commit"
18808            );
18809        }
18810    }
18811
18812    #[tokio::test]
18813    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18814        crate::run::pin_test_home();
18815        let tmp = tempfile::tempdir().expect("tempdir");
18816        let repo = tmp.path().join("repo");
18817        std::fs::create_dir_all(&repo).unwrap();
18818        init_repo(&repo);
18819
18820        let mut config = Config::default();
18821        config.merge.mode = MergeMode::Pr;
18822        config.graph.land = true;
18823        config.graph.land_approval = false;
18824
18825        let mut state = RunState::new(
18826            repo.clone(),
18827            "main".to_owned(),
18828            "deadbeef".to_owned(),
18829            "task".to_owned(),
18830            config,
18831        );
18832        state.candidates = vec![Candidate {
18833            index: 0,
18834            label: 'A',
18835            agent: "alpha".to_owned(),
18836            branch: "does-not-exist".to_owned(),
18837            worktree: repo.clone(),
18838            summary: String::new(),
18839            stat: String::new(),
18840            files: 0,
18841            commits: 0,
18842            empty: false,
18843            failed: None,
18844            verified_noop: None,
18845            duration_ms: 0,
18846            folded: false,
18847        }];
18848        state.tally = Some(Tally {
18849            first_choice: BTreeMap::from([('A', 1)]),
18850            borda: BTreeMap::new(),
18851            winner: 'A',
18852            rankings: 1,
18853            unanimous_initial: true,
18854            deliberated: false,
18855            changed_votes: 0,
18856            unanimous_final: true,
18857            tie_break: None,
18858            judges: 0,
18859            present: 0,
18860            quorum: 0,
18861            met_quorum: true,
18862            uncontested: Some("only candidate A produced a change".to_owned()),
18863        });
18864        state.reviews = vec![ReviewRound {
18865            round: 1,
18866            head: "deadbeef".to_owned(),
18867            verified_head: None,
18868            verified_at: None,
18869            reviews: Vec::new(),
18870            e2e: Vec::new(),
18871            fix: None,
18872            blocking: 0,
18873            answered: 0,
18874            expected: 0,
18875            clean: true,
18876            verify_retried: false,
18877            e2e_deferred: false,
18878            e2e_defer_reason: None,
18879            progressed: false,
18880            vote_split: false,
18881            reconsideration: Vec::new(),
18882            verdict: None,
18883        }];
18884        state.gate = vec![CommandOutcome {
18885            command: "test".to_owned(),
18886            code: Some(0),
18887            output_tail: String::new(),
18888            duration_ms: 0,
18889            resource_blocked: false,
18890        }];
18891        state.gate_ran = true;
18892        // A first pass through `merge` already pushed and opened this pull
18893        // request; `status` is `Landing` because a previous call into `land`
18894        // parked or was interrupted before it reached a terminal outcome.
18895        state.status = RunStatus::Landing;
18896        state.merge = Some(MergeOutcome {
18897            mode: MergeMode::Pr,
18898            ok: true,
18899            detail: "https://example.invalid/x/y/pull/1".to_owned(),
18900            empty: false,
18901        });
18902
18903        // The Landing-resume shortcut calls `run_land` directly rather than
18904        // through `merge`, which is exactly the call site that used to skip
18905        // `settle_questions` - see the fixture below.
18906        ask_test_home();
18907        // A real resume loads a persisted run; driver ownership is checked
18908        // against that record before the Landing shortcut is entered.
18909        state.save().expect("save landing run");
18910        let store = ask::Questions::open();
18911        let q = ask_open_question(&store, &state.id);
18912
18913        let mut runner = Runner {
18914            state,
18915            roles: ResolvedRoles {
18916                implementers: Vec::new(),
18917                judges: Vec::new(),
18918                reviewers: Vec::new(),
18919                fixer: None,
18920                conductor: conductor(),
18921                implementer_roster: Vec::new(),
18922                judge_roster: Vec::new(),
18923                reviewer_roster: Vec::new(),
18924            },
18925            sem: Arc::new(Semaphore::new(1)),
18926            pause: Pause::new(),
18927            interrupt: Pause::new(),
18928        };
18929
18930        // `execute`, not `merge` directly: the Landing-resume shortcut lives
18931        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
18932        // exactly because `review_loop` would otherwise clobber the marker
18933        // first.
18934        runner.execute().await.expect("execute");
18935
18936        assert_eq!(
18937            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
18938            Some("https://example.invalid/x/y/pull/1"),
18939            "reentry must not push again or open a second pull request over the \
18940             one `land` is already watching"
18941        );
18942        assert_ne!(
18943            runner.state.status,
18944            RunStatus::Landing,
18945            "land could not actually reach the fake pull request, so it must \
18946             have given up rather than left the run silently parked forever"
18947        );
18948        // `land` could not reach the fake pull request, so it gave up into
18949        // `Blocked` - still resumable, so the question must not have been
18950        // swept just because this branch now also calls `settle_questions`.
18951        assert_eq!(runner.state.status, RunStatus::Blocked);
18952        assert!(
18953            store.get(&q.id).unwrap().status.open(),
18954            "Blocked is still alive; settle_questions must have been a no-op here"
18955        );
18956    }
18957
18958    fn state_with_round(round: ReviewRound) -> RunState {
18959        let mut s = RunState::new(
18960            PathBuf::from("/repo"),
18961            "main".to_owned(),
18962            "abc1234".to_owned(),
18963            "add retries".to_owned(),
18964            Config::default(),
18965        );
18966        s.reviews = vec![round];
18967        s
18968    }
18969
18970    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
18971        crate::verdict::Finding {
18972            id: id.to_owned(),
18973            severity,
18974            file: None,
18975            line: None,
18976            title: title.to_owned(),
18977            detail: String::new(),
18978        }
18979    }
18980
18981    #[test]
18982    fn pr_body_names_open_findings_and_declined_ones() {
18983        let round = ReviewRound {
18984            round: 2,
18985            head: "deadbee".to_owned(),
18986            verified_head: None,
18987            verified_at: None,
18988            reviews: vec![ReviewRecord {
18989                attempts: 0,
18990                reviewer: 1,
18991                agent: "alpha".to_owned(),
18992                summary: String::new(),
18993                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
18994                vote: None,
18995                failed: None,
18996                duration_ms: 0,
18997            }],
18998            e2e: vec![CommandOutcome {
18999                command: "cargo test".to_owned(),
19000                code: Some(0),
19001                output_tail: String::new(),
19002                duration_ms: 0,
19003                resource_blocked: false,
19004            }],
19005            verify_retried: false,
19006            e2e_deferred: false,
19007            e2e_defer_reason: None,
19008            fix: Some(FixRecord {
19009                agent: "alpha".to_owned(),
19010                addressed: Vec::new(),
19011                rejected: vec![crate::verdict::Rejection {
19012                    id: "R1-1-1".to_owned(),
19013                    why: "not reachable from any caller".to_owned(),
19014                }],
19015                notes: String::new(),
19016                committed: true,
19017                failed: None,
19018                duration_ms: 0,
19019                continuation: None,
19020            }),
19021            blocking: 0,
19022            answered: 1,
19023            expected: 1,
19024            clean: false,
19025            progressed: true,
19026            vote_split: false,
19027            reconsideration: Vec::new(),
19028            verdict: None,
19029        };
19030        let state = state_with_round(round);
19031        let body = pr_message(&state, 'A').body;
19032
19033        assert!(body.contains("add retries"), "the task must still be there");
19034        assert!(body.contains("R2-1-1"), "{body}");
19035        assert!(!body.contains("unused import"), "{body}");
19036        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19037        assert!(
19038            !body.contains("not reachable from any caller"),
19039            "the reason it was declined: {body}"
19040        );
19041    }
19042
19043    #[test]
19044    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19045        let round = ReviewRound {
19046            round: 1,
19047            head: "deadbee".to_owned(),
19048            verified_head: None,
19049            verified_at: None,
19050            reviews: vec![ReviewRecord {
19051                attempts: 0,
19052                reviewer: 1,
19053                agent: "alpha".to_owned(),
19054                summary: String::new(),
19055                findings: Vec::new(),
19056                vote: None,
19057                failed: None,
19058                duration_ms: 0,
19059            }],
19060            e2e: Vec::new(),
19061            verify_retried: false,
19062            e2e_deferred: false,
19063            e2e_defer_reason: None,
19064            fix: None,
19065            blocking: 0,
19066            answered: 1,
19067            expected: 1,
19068            clean: true,
19069            progressed: false,
19070            vote_split: false,
19071            reconsideration: Vec::new(),
19072            verdict: None,
19073        };
19074        let state = state_with_round(round);
19075        let body = pr_message(&state, 'A').body;
19076        assert!(!body.contains("Open review findings"), "{body}");
19077        assert!(!body.contains("Declined"), "{body}");
19078    }
19079
19080    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19081        let mut state = RunState::new(
19082            PathBuf::from("/repo"),
19083            "main".to_owned(),
19084            "abc1234".to_owned(),
19085            instruction.to_owned(),
19086            Config::default(),
19087        );
19088        state.candidates.push(Candidate {
19089            index: 0,
19090            label: 'A',
19091            agent: "alpha".to_owned(),
19092            branch: "magi/x/A".to_owned(),
19093            worktree: PathBuf::from("/wt"),
19094            summary: summary.to_owned(),
19095            stat: String::new(),
19096            files: 1,
19097            commits: 1,
19098            empty: false,
19099            failed: None,
19100            verified_noop: None,
19101            folded: false,
19102            duration_ms: 0,
19103        });
19104        state
19105    }
19106
19107    fn review_state(subjects: &[&str]) -> RunState {
19108        let mut state = state_with_summary(
19109            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19110            "",
19111        );
19112        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19113        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19114        state
19115    }
19116
19117    /// A branch rebased onto a main that moved past the recorded
19118    /// `base_commit` is titled from its own first commit, never main's.
19119    #[tokio::test]
19120    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19121        ask_test_home();
19122        let tmp = tempfile::tempdir().unwrap();
19123        let repo = tmp.path().join("repo");
19124        std::fs::create_dir_all(&repo).unwrap();
19125        init_repo(&repo);
19126        let origin = tmp.path().join("origin.git");
19127        let g = |dir: &Path, args: &[&str]| {
19128            let out = std::process::Command::new("git")
19129                .args(args)
19130                .current_dir(dir)
19131                .quiet()
19132                .output()
19133                .expect("spawn git");
19134            assert!(
19135                out.status.success(),
19136                "git {args:?}: {}",
19137                String::from_utf8_lossy(&out.stderr)
19138            );
19139        };
19140        g(
19141            tmp.path(),
19142            &[
19143                "clone",
19144                "--bare",
19145                "-q",
19146                repo.to_str().unwrap(),
19147                origin.to_str().unwrap(),
19148            ],
19149        );
19150        g(
19151            &repo,
19152            &["remote", "add", "origin", origin.to_str().unwrap()],
19153        );
19154        let c1 = git::rev_parse(&repo, "main").await.unwrap();
19155
19156        // Main moves on; the branch is built on top of the new main.
19157        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19158        g(&repo, &["add", "-A"]);
19159        g(
19160            &repo,
19161            &["commit", "-q", "-m", "chore(deps): update a crate"],
19162        );
19163        g(&repo, &["push", "-q", "origin", "main"]);
19164        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19165        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19166        g(&repo, &["add", "-A"]);
19167        g(
19168            &repo,
19169            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19170        );
19171        g(&repo, &["checkout", "-q", "main"]);
19172
19173        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19174        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19175        // Without a readable tracking ref the recorded base's merge base is used.
19176        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19177        assert_eq!(fallback, c1);
19178
19179        let mut state = review_state(&[
19180            "chore(deps): update a crate",
19181            "fix(daemon): apply a chosen action",
19182        ]);
19183        state.repo = repo.clone();
19184        state.base_branch = "main".to_owned();
19185        state.base_commit = c1;
19186        refresh_reviewed_commits(&mut state, "feat/own").await;
19187        assert_eq!(
19188            state.reviewed_commits,
19189            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19190        );
19191        assert_eq!(
19192            review_title(&state).as_deref(),
19193            Some("fix(daemon): apply a chosen action")
19194        );
19195        assert_eq!(
19196            leaked_subjects(&state, "feat/own").await,
19197            Some(vec!["chore(deps): update a crate".to_owned()])
19198        );
19199        // A stale tracking ref is still used when the fetch fails, but the
19200        // leak list is withheld.
19201        state.config.merge.remote = "nowhere".to_owned();
19202        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19203    }
19204
19205    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
19206    /// with `origin/main` one commit behind it, and a summarizing agent that
19207    /// prints whatever `reply` holds.
19208    struct SummaryFixture {
19209        _tmp: tempfile::TempDir,
19210        repo: PathBuf,
19211        reply: PathBuf,
19212        runner: Runner,
19213    }
19214
19215    fn summary_fixture() -> SummaryFixture {
19216        ask_test_home();
19217        let tmp = tempfile::tempdir().unwrap();
19218        let repo = tmp.path().join("repo");
19219        std::fs::create_dir_all(&repo).unwrap();
19220        init_repo(&repo);
19221        let origin = tmp.path().join("origin.git");
19222        let g = |args: &[&str]| {
19223            let out = std::process::Command::new("git")
19224                .args(args)
19225                .current_dir(&repo)
19226                .quiet()
19227                .output()
19228                .expect("spawn git");
19229            assert!(
19230                out.status.success(),
19231                "git {args:?}: {}",
19232                String::from_utf8_lossy(&out.stderr)
19233            );
19234        };
19235        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19236        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19237        g(&["push", "-q", "origin", "main"]);
19238        g(&["checkout", "-q", "-b", "feat/own"]);
19239        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19240        g(&["add", "-A"]);
19241        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19242        let reply = tmp.path().join("reply.txt");
19243        std::fs::write(
19244            &reply,
19245            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19246        )
19247        .unwrap();
19248        let mut runner = runner_at(RunStatus::Gating);
19249        let mut state = review_state(&["feat(own): add the own file"]);
19250        state.repo = repo.clone();
19251        state.base_branch = "main".to_owned();
19252        state.candidates[0].branch = "feat/own".to_owned();
19253        state.candidates[0].worktree = repo.clone();
19254        let mut author = spec("alpha");
19255        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19256        author
19257            .env
19258            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19259        state.config.agents = vec![author];
19260        runner.state = state;
19261        SummaryFixture {
19262            _tmp: tmp,
19263            repo,
19264            reply,
19265            runner,
19266        }
19267    }
19268
19269    async fn posted(f: &mut SummaryFixture) -> PrMessage {
19270        let winner = f.runner.state.candidates[0].clone();
19271        f.runner
19272            .guarded_pr_message(&winner, None, true)
19273            .await
19274            .unwrap()
19275            .expect("an English description never parks")
19276    }
19277
19278    fn clean_round_with_minor_findings() -> ReviewRound {
19279        let mut round = passing_round(2, None);
19280        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19281        round.reviews = vec![ReviewRecord {
19282            attempts: 0,
19283            reviewer: 1,
19284            agent: "alpha".into(),
19285            summary: "内部レビュー".into(),
19286            findings: vec![
19287                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19288                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19289            ],
19290            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19291            failed: None,
19292            duration_ms: 0,
19293        }];
19294        round
19295    }
19296
19297    #[tokio::test]
19298    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19299        for review_only in [false, true] {
19300            let mut f = summary_fixture();
19301            if !review_only {
19302                f.runner.state.candidates[0].agent = "alpha".into();
19303            }
19304            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19305            assert!(f.runner.state.reviews[0].clean);
19306            assert!(f.runner.state.reviews[0].fix.is_none());
19307            assert!(f.runner.state.open_findings().is_empty());
19308            assert_eq!(
19309                crate::pr_outcome::required_finding_ids(&f.runner.state),
19310                ["R2-1-1", "R2-1-2"]
19311            );
19312            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19313            let evidence = crate::pr_outcome::summary_evidence(
19314                &f.runner.state,
19315                crate::pr_outcome::FinalHead::At(&head),
19316            );
19317            assert!(
19318                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19319            );
19320            // The ordinary reply omits both IDs: no cached description is adopted.
19321            let omitted = posted(&mut f).await;
19322            assert!(omitted.body.contains("No description"));
19323            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19324            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19325            assert!(f.runner.state.change_summary.is_none());
19326            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19327            let described = posted(&mut f).await;
19328            assert!(described.body.contains("improve documentation for callers"));
19329            assert!(described.body.contains("remaining edge case"));
19330            assert!(described.body.contains("## Open review findings"));
19331            assert!(!described.body.contains("未修正"));
19332        }
19333    }
19334
19335    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19336        let mut f = summary_fixture();
19337        if !review_only {
19338            f.runner.state.candidates[0].agent = "alpha".into();
19339        }
19340        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19341        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19342        fixed
19343            .fix
19344            .as_mut()
19345            .unwrap()
19346            .rejected
19347            .push(crate::verdict::Rejection {
19348                id: "R1-1-1".into(),
19349                why: "呼び出されないコードのため辞退".into(),
19350            });
19351        let mut clean = clean_round_with_minor_findings();
19352        clean.verified_head = Some(head.clone());
19353        f.runner.state.reviews = vec![fixed, clean];
19354        ok_gate(&mut f.runner.state, Some(&head));
19355        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19356        f.runner.state.config.agents.push(scripted("beta", reply));
19357        f.runner.state.config.roles.language_judge =
19358            Some(crate::config::AgentChoice::One("beta".into()));
19359        f
19360    }
19361
19362    #[tokio::test]
19363    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19364        for review_only in [false, true] {
19365            let mut f = rewrite_fixture_with_findings(review_only,
19366                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19367            let m = posted(&mut f).await;
19368            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19369            let expected =
19370                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19371            assert!(m.body.contains(&expected));
19372            assert!(m.body.contains("## Review rounds"));
19373            assert!(
19374                m.body
19375                    .contains("Gate (`verify.gate`): passed on the final commit")
19376            );
19377            assert!(
19378                m.body
19379                    .contains("Tests (`verify.e2e`): passed on the final commit")
19380            );
19381            let c = f.runner.state.change_summary.as_ref().unwrap();
19382            assert!(
19383                m.body
19384                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19385            );
19386            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19387            assert!(crate::pr_outcome::covers_required_findings(
19388                &f.runner.state,
19389                &m.body
19390            ));
19391            assert!(m.body.contains("documentation remains incomplete"));
19392            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19393            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19394            assert!(f.runner.state.github_text.is_none());
19395        }
19396    }
19397
19398    #[tokio::test]
19399    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19400        for review_only in [false, true] {
19401            for reply in [
19402                "TITLE: fix: public description\nA short English description.",
19403                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19404            ] {
19405                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19406                let winner = f.runner.state.candidates[0].clone();
19407                let posted = f
19408                    .runner
19409                    .guarded_pr_message(&winner, None, true)
19410                    .await
19411                    .unwrap();
19412                assert!(posted.is_none());
19413                assert!(f.runner.state.parked);
19414                let withheld = f.runner.state.github_text.as_ref().unwrap();
19415                assert!(
19416                    withheld.rewrite.is_none(),
19417                    "an incomplete rewrite must not be selectable"
19418                );
19419                let q = ask::Questions::open()
19420                    .get(withheld.question.as_ref().unwrap())
19421                    .unwrap();
19422                assert!(
19423                    !q.choices
19424                        .contains(&crate::github_text::USE_REWRITE.to_owned())
19425                );
19426                assert!(
19427                    q.detail.contains("R1-1-1")
19428                        && q.detail.contains("R2-1-1")
19429                        && q.detail.contains("R2-1-2")
19430                );
19431            }
19432        }
19433    }
19434
19435    #[tokio::test]
19436    async fn japanese_reports_stay_local_in_both_public_run_modes() {
19437        for review_only in [false, true] {
19438            let mut f = summary_fixture();
19439            if !review_only {
19440                f.runner.state.candidates[0].agent = "alpha".into();
19441            }
19442            f.runner.state.instruction = "内部タスクの説明".into();
19443            f.runner.state.candidates[0].summary = "内部の候補報告".into();
19444            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19445            f.runner.state.reviews = vec![
19446                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
19447                round_with_notes(2, Some("二回目の内部修正報告です。")),
19448                passing_round(3, Some(&head)),
19449            ];
19450            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
19451            ok_gate(&mut f.runner.state, Some(&head));
19452            let captured = f.repo.join("public-prompt.txt");
19453            let author = &mut f.runner.state.config.agents[0];
19454            author.command = vec![
19455                "sh".into(),
19456                "-c".into(),
19457                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19458            ];
19459            author.env.insert(
19460                "PROMPT_FILE".into(),
19461                captured.to_string_lossy().into_owned(),
19462            );
19463            let m = posted(&mut f).await;
19464            assert!(m.body.contains("Adds `own.txt`"));
19465            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
19466            assert!(m.body.contains("R1-1-1"));
19467            assert!(
19468                m.body
19469                    .contains("Gate (`verify.gate`): passed on the final commit")
19470            );
19471            assert!(
19472                m.body
19473                    .contains("Tests (`verify.e2e`): passed on the final commit")
19474            );
19475            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19476            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
19477            let prompt = std::fs::read_to_string(captured).unwrap();
19478            assert!(prompt.contains(&format!(
19479                "git diff {} {head}",
19480                f.runner.state.change_summary.as_ref().unwrap().base
19481            )));
19482            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
19483            assert_eq!(
19484                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
19485                "二回目の内部修正報告です。"
19486            );
19487        }
19488    }
19489
19490    #[tokio::test]
19491    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
19492        for review_only in [false, true] {
19493            let mut f = summary_fixture();
19494            if !review_only {
19495                f.runner.state.candidates[0].agent = "alpha".into();
19496            }
19497            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
19498            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
19499            std::fs::write(&f.reply, "no usable description").unwrap();
19500            let failed = posted(&mut f).await;
19501            assert!(
19502                failed
19503                    .body
19504                    .contains("No description of what this change does could be generated")
19505            );
19506            assert!(failed.body.is_ascii());
19507            assert!(f.runner.state.github_text.is_none());
19508            assert!(f.runner.state.change_summary.is_none());
19509            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19510            let evidence = crate::pr_outcome::summary_evidence(
19511                &f.runner.state,
19512                crate::pr_outcome::FinalHead::At(&head),
19513            );
19514            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
19515            assert!(
19516                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19517                    .exists()
19518            );
19519            f.runner.state =
19520                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19521            std::fs::write(
19522                &f.reply,
19523                "TITLE: feat(own): add the own file\nAdds the committed own file.",
19524            )
19525            .unwrap();
19526            let retried = posted(&mut f).await;
19527            assert!(retried.body.contains("Adds the committed own file."));
19528            assert_eq!(
19529                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19530                "日本語の修正報告"
19531            );
19532        }
19533    }
19534
19535    #[tokio::test]
19536    async fn same_head_verification_and_findings_invalidate_public_prose() {
19537        let mut f = summary_fixture();
19538        posted(&mut f).await;
19539        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19540        let original = f
19541            .runner
19542            .state
19543            .change_summary
19544            .as_ref()
19545            .unwrap()
19546            .evidence
19547            .clone();
19548        ok_gate(&mut f.runner.state, Some(&head));
19549        std::fs::write(
19550            &f.reply,
19551            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19552        )
19553        .unwrap();
19554        let verified = posted(&mut f).await;
19555        assert!(verified.body.contains("now verified"));
19556        assert_ne!(
19557            f.runner.state.change_summary.as_ref().unwrap().evidence,
19558            original
19559        );
19560        let mut round = round_with_notes(1, Some("内部報告"));
19561        round.reviews.push(ReviewRecord {
19562            attempts: 0,
19563            reviewer: 1,
19564            agent: "alpha".into(),
19565            summary: "内部レビュー".into(),
19566            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19567            vote: None,
19568            failed: None,
19569            duration_ms: 0,
19570        });
19571        round
19572            .fix
19573            .as_mut()
19574            .unwrap()
19575            .rejected
19576            .push(crate::verdict::Rejection {
19577                id: "R1-1-1".into(),
19578                why: "呼び出し元から到達できない".into(),
19579            });
19580        f.runner.state.reviews = vec![round];
19581        // A description omitting the required ID is unusable, even with a title.
19582        let omitted = posted(&mut f).await;
19583        assert!(omitted.body.contains("No description"));
19584        assert!(omitted.body.contains("R1-1-1"));
19585        assert!(!omitted.body.contains("到達"));
19586        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19587        let described = posted(&mut f).await;
19588        assert!(described.body.contains("unreachable from callers"));
19589        assert!(described.body.contains("R1-1-1"));
19590        assert!(!described.body.contains("到達"));
19591        let evidence = f
19592            .runner
19593            .state
19594            .change_summary
19595            .as_ref()
19596            .unwrap()
19597            .evidence
19598            .clone();
19599        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19600        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19601        assert!(posted(&mut f).await.body.contains("rationale updated"));
19602        assert_ne!(
19603            f.runner.state.change_summary.as_ref().unwrap().evidence,
19604            evidence
19605        );
19606        // Old serialized summaries have no evidence fingerprint and must regenerate.
19607        f.runner
19608            .state
19609            .change_summary
19610            .as_mut()
19611            .unwrap()
19612            .evidence
19613            .clear();
19614        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19615        assert!(posted(&mut f).await.body.contains("current specification"));
19616    }
19617
19618    #[tokio::test]
19619    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19620        let mut f = summary_fixture();
19621        let m = posted(&mut f).await;
19622        assert_eq!(m.title, "feat(own): add the own file");
19623        assert!(
19624            m.body
19625                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19626            "{}",
19627            m.body
19628        );
19629        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19630        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19631        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19632        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19633        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19634        let kept = f.runner.state.change_summary.as_ref().unwrap();
19635        assert_eq!(
19636            (kept.head.as_str(), kept.base.as_str()),
19637            (head.as_str(), base.as_str())
19638        );
19639        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19640    }
19641
19642    #[tokio::test]
19643    async fn review_only_summary_receives_the_configured_github_language() {
19644        let mut f = summary_fixture();
19645        f.runner.state.config.graph.github_language = "ja".into();
19646        let captured = f.repo.join("summary-prompt.txt");
19647        let author = &mut f.runner.state.config.agents[0];
19648        author.command = vec![
19649            "sh".into(),
19650            "-c".into(),
19651            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19652        ];
19653        author.env.insert(
19654            "PROMPT_FILE".into(),
19655            captured.to_string_lossy().into_owned(),
19656        );
19657        std::fs::write(
19658            &f.reply,
19659            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19660        )
19661        .unwrap();
19662        let m = posted(&mut f).await;
19663        let prompt = std::fs::read_to_string(captured).unwrap();
19664        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19665        assert!(!prompt.contains("in English"));
19666        assert_eq!(m.title, "feat(own): ファイルを追加");
19667        assert!(m.body.contains("own.txt を追加します。"));
19668    }
19669
19670    #[tokio::test]
19671    async fn review_fixes_come_after_the_whole_change_summary() {
19672        let mut f = summary_fixture();
19673        f.runner.state.reviews = vec![
19674            round_with_notes(1, Some("handled the PATH case")),
19675            round_with_notes(2, None),
19676        ];
19677        let m = posted(&mut f).await;
19678        let summary = m.body.find("## Summary").expect("summary");
19679        let fixes = m.body.find("## Review rounds").expect("review rounds");
19680        assert!(summary < fixes, "{}", m.body);
19681        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19682        assert!(m.body.contains("## Final verification"), "{}", m.body);
19683    }
19684
19685    #[tokio::test]
19686    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19687        let mut f = summary_fixture();
19688        let first = posted(&mut f).await;
19689        let first_head = f.runner.state.change_summary.clone().unwrap().head;
19690        // The same head reuses the record: a different reply is not read.
19691        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19692        let again = posted(&mut f).await;
19693        assert_eq!((again.title, again.body), (first.title, first.body));
19694
19695        let g = |args: &[&str]| {
19696            let out = std::process::Command::new("git")
19697                .args(args)
19698                .current_dir(&f.repo)
19699                .quiet()
19700                .output()
19701                .expect("spawn git");
19702            assert!(out.status.success(), "git {args:?}");
19703        };
19704        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19705        g(&["add", "-A"]);
19706        g(&["commit", "-q", "-m", "fix(own): handle more"]);
19707        let second = posted(&mut f).await;
19708        let kept = f.runner.state.change_summary.clone().unwrap();
19709        assert_ne!(kept.head, first_head);
19710        assert!(second.body.contains("Not read yet."), "{}", second.body);
19711        assert!(!second.body.contains("so the first summary is read"));
19712        assert_eq!(second.title, "feat(own): other");
19713    }
19714
19715    #[tokio::test]
19716    async fn an_unavailable_summary_is_stated_and_never_invented() {
19717        let mut f = summary_fixture();
19718        std::fs::write(&f.reply, "no title line here").unwrap();
19719        let m = posted(&mut f).await;
19720        assert!(
19721            m.body
19722                .contains("No description of what this change does could be generated"),
19723            "{}",
19724            m.body
19725        );
19726        assert!(m.body.contains("internal reports remain in local records"));
19727        assert!(f.runner.state.change_summary.is_none());
19728        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19729        // A preview asks nobody and claims nothing.
19730        let winner = f.runner.state.candidates[0].clone();
19731        let preview = f
19732            .runner
19733            .guarded_pr_message(&winner, None, false)
19734            .await
19735            .unwrap()
19736            .unwrap();
19737        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19738    }
19739
19740    #[test]
19741    fn pr_message_review_single_commit_uses_its_subject() {
19742        let state = review_state(&["feat(nats): per-role user"]);
19743        let m = pr_message(&state, 'A');
19744        assert_eq!(m.title, "feat(nats): per-role user");
19745        assert!(!m.body.contains("Review the work already"), "{}", m.body);
19746        assert!(m.body.contains("## Commits under review"), "{}", m.body);
19747    }
19748
19749    #[test]
19750    fn pr_message_review_multi_commit_takes_the_oldest() {
19751        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19752        let m = pr_message(&state, 'A');
19753        assert_eq!(m.title, "feat: the change");
19754        for s in ["feat: the change", "fix: typo", "fix: again"] {
19755            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19756        }
19757    }
19758
19759    #[test]
19760    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19761        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19762            let mut state = review_state(&[first, "fix: later fixup"]);
19763            state.candidates[0].branch = "機能/ログイン".to_owned();
19764            let m = pr_message(&state, 'A');
19765            assert!(
19766                m.title.starts_with("chore: land candidate A of run"),
19767                "{}",
19768                m.title
19769            );
19770        }
19771    }
19772
19773    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19774        BranchFacts {
19775            commits: commits
19776                .iter()
19777                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19778                .collect(),
19779            stat: stat.to_owned(),
19780        }
19781    }
19782
19783    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19784        let mut r = review_round(true, 0, 1, 1, true, true);
19785        r.round = round;
19786        r.fix = notes.map(|n| FixRecord {
19787            agent: "fixer".to_owned(),
19788            addressed: Vec::new(),
19789            rejected: Vec::new(),
19790            notes: n.to_owned(),
19791            committed: true,
19792            failed: None,
19793            duration_ms: 0,
19794            continuation: None,
19795        });
19796        r
19797    }
19798
19799    #[test]
19800    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19801        let state = review_state(&["ignored"]);
19802        let f = facts(
19803            &[
19804                (
19805                    "fix(login): resolve PATH on macOS",
19806                    "Login shells skip rc files.",
19807                ),
19808                ("fix: address review", ""),
19809            ],
19810            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19811        );
19812        let m = pr_message_with(&state, 'A', Some(&f));
19813        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19814        assert!(
19815            m.body
19816                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
19817            "{}",
19818            m.body
19819        );
19820        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19821        assert!(m.body.contains("## Diff stat"), "{}", m.body);
19822        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19823        for banned in [
19824            "Review the work already",
19825            "no task statement",
19826            "Original task",
19827        ] {
19828            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19829        }
19830        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19831    }
19832
19833    #[test]
19834    fn pr_message_review_truncates_a_huge_first_commit_body() {
19835        let state = review_state(&["ignored"]);
19836        let f = facts(
19837            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19838            "s",
19839        );
19840        let m = pr_message_with(&state, 'A', Some(&f));
19841        assert!(m.body.len() < 30_000, "{}", m.body.len());
19842        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19843        assert!(
19844            m.body.contains("- ... 1 more commit(s)"),
19845            "{}",
19846            m.body.len()
19847        );
19848        assert!(m.body.ends_with("magi:candidate-a\n"));
19849    }
19850
19851    #[test]
19852    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19853        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19854        assert_eq!(m.title, "feat: x");
19855        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19856        assert!(!m.body.contains("Diff stat"), "{}", m.body);
19857        assert!(!m.body.contains("no task statement"), "{}", m.body);
19858    }
19859
19860    #[test]
19861    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19862        let mut state = review_state(&["日本語の件名"]);
19863        state.candidates[0].branch = "fix/macos-login-path".to_owned();
19864        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19865        state.candidates[0].branch = "機能/ログイン".to_owned();
19866        assert!(
19867            pr_message(&state, 'A')
19868                .title
19869                .starts_with("chore: land candidate A")
19870        );
19871    }
19872
19873    #[test]
19874    fn pr_message_review_fixes_survive_a_clean_final_round() {
19875        let mut state = review_state(&["feat: x"]);
19876        state.reviews = vec![
19877            round_with_notes(1, Some("handled the PATH case")),
19878            round_with_notes(2, None),
19879        ];
19880        let body = pr_message(&state, 'A').body;
19881        assert!(
19882            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19883            "{body}"
19884        );
19885        assert!(body.contains("### Round 1"), "{body}");
19886
19887        state.reviews = vec![
19888            round_with_notes(1, Some("first fix")),
19889            round_with_notes(2, Some("")),
19890            round_with_notes(3, Some("second fix")),
19891            round_with_notes(4, None),
19892        ];
19893        let body = pr_message(&state, 'A').body;
19894        assert!(
19895            body.contains("### Round 1") && !body.contains("first fix"),
19896            "{body}"
19897        );
19898        assert!(
19899            body.contains("### Round 3") && !body.contains("second fix"),
19900            "{body}"
19901        );
19902        assert!(body.contains("### Round 2"), "{body}");
19903    }
19904
19905    const FINAL: &str = "abcdef1234567890";
19906
19907    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
19908        let mut r = round_with_notes(round, None);
19909        r.verified_head = verified.map(str::to_owned);
19910        r
19911    }
19912
19913    fn ok_gate(state: &mut RunState, head: Option<&str>) {
19914        state.gate = vec![CommandOutcome {
19915            command: "gate".to_owned(),
19916            code: Some(0),
19917            output_tail: String::new(),
19918            duration_ms: 0,
19919            resource_blocked: false,
19920        }];
19921        state.gate_ran = true;
19922        state.gate_head = head.map(str::to_owned);
19923    }
19924
19925    fn body_at(state: &RunState) -> String {
19926        pr_message_raw(
19927            state,
19928            'A',
19929            None,
19930            crate::pr_outcome::FinalHead::At(FINAL),
19931            ChangeNote::NotAsked,
19932        )
19933        .body
19934    }
19935
19936    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
19937
19938    #[test]
19939    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
19940        let mut state = review_state(&["feat: x"]);
19941        state.reviews = vec![
19942            round_with_notes(1, Some(SANDBOXED)),
19943            passing_round(2, Some(FINAL)),
19944        ];
19945        ok_gate(&mut state, Some(FINAL));
19946        let body = body_at(&state);
19947        assert!(!body.contains("Handled the PATH case."), "{body}");
19948        assert!(!body.contains("could not commit"), "{body}");
19949        assert!(!body.contains("could not run"), "{body}");
19950        assert!(
19951            body.contains("The fix is committed on the branch"),
19952            "{body}"
19953        );
19954        assert!(!body.contains("sandbox"), "{body}");
19955        assert!(
19956            body.contains("Tests (`verify.e2e`): passed on the final commit"),
19957            "{body}"
19958        );
19959        assert!(
19960            body.contains("Gate (`verify.gate`): passed on the final commit"),
19961            "{body}"
19962        );
19963        // The raw report is untouched in the record.
19964        assert!(
19965            state.reviews[0]
19966                .fix
19967                .as_ref()
19968                .unwrap()
19969                .notes
19970                .contains("could not commit")
19971        );
19972    }
19973
19974    #[test]
19975    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
19976        let mut state = review_state(&["feat: x"]);
19977        let mut r1 = round_with_notes(1, Some(SANDBOXED));
19978        r1.fix.as_mut().unwrap().committed = false;
19979        let mut r2 = passing_round(2, Some(FINAL));
19980        r2.e2e[0].code = Some(1);
19981        state.reviews = vec![r1, r2];
19982        let body = body_at(&state);
19983        assert!(!body.contains("could not commit"), "{body}");
19984        assert!(!body.contains("could not run"), "{body}");
19985        assert!(body.contains("This round produced no commit"), "{body}");
19986        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
19987        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
19988        assert!(!body.contains("passed"), "{body}");
19989    }
19990
19991    #[test]
19992    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
19993        let mut state = review_state(&["feat: x"]);
19994        state.reviews = vec![
19995            round_with_notes(1, Some(SANDBOXED)),
19996            passing_round(2, Some("0123456789abcdef")),
19997        ];
19998        ok_gate(&mut state, Some("0123456789abcdef"));
19999        let body = body_at(&state);
20000        assert!(!body.contains("HTTP tests could not run"), "{body}");
20001        // The commit claim is still settled by the structured record.
20002        assert!(!body.contains("could not commit"), "{body}");
20003        assert!(
20004            body.contains("passed on an earlier commit (`0123456`)"),
20005            "{body}"
20006        );
20007        assert!(!body.contains("passed on the final commit"), "{body}");
20008
20009        // A gate recorded before `gate_head` existed is "not recorded".
20010        ok_gate(&mut state, None);
20011        let body = body_at(&state);
20012        assert!(
20013            body.contains("passed on a commit that was not recorded"),
20014            "{body}"
20015        );
20016    }
20017
20018    #[test]
20019    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20020        let state = state_with_summary(
20021            "add retries to the client",
20022            "TITLE: feat: retries\n\nDid it.",
20023        );
20024        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20025        assert_eq!(m.title, "feat: retries");
20026        assert!(
20027            m.body.contains("<summary>Original task</summary>"),
20028            "{}",
20029            m.body
20030        );
20031        assert!(!m.body.contains("Commits under review"), "{}", m.body);
20032        assert!(
20033            m.body
20034                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20035            "{}",
20036            m.body
20037        );
20038    }
20039
20040    #[test]
20041    fn pr_message_review_bounds_a_long_english_subject() {
20042        let long = format!("feat: {}", "word ".repeat(100));
20043        let m = pr_message(&review_state(&[&long]), 'A');
20044        assert!(m.title.starts_with("feat: word"), "{}", m.title);
20045        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20046    }
20047
20048    #[test]
20049    fn pr_message_implementation_run_is_unchanged_by_review_support() {
20050        let state = state_with_summary("add retries\n\ndetails", "- did some things");
20051        let m = pr_message(&state, 'A');
20052        assert_eq!(m.title, "add retries");
20053        assert!(m.body.contains("<summary>Original task</summary>"));
20054        assert!(!m.body.contains("Commits under review"));
20055        assert_eq!(landing_subject_source(&state), state.instruction);
20056    }
20057
20058    #[test]
20059    fn review_run_squash_subject_is_the_change_not_the_prompt() {
20060        let state = review_state(&["feat: the change", "fix: typo"]);
20061        let source = landing_subject_source(&state);
20062        assert_eq!(land::merge_subject("", &source), "feat: the change");
20063        assert_eq!(
20064            land::merge_subject("magi: candidate A (uncommitted work)", &source),
20065            "feat: the change"
20066        );
20067        // An operator's rename still wins.
20068        assert_eq!(
20069            land::merge_subject("feat: renamed by hand", &source),
20070            "feat: renamed by hand"
20071        );
20072        let blank = review_state(&["日本語"]);
20073        assert!(
20074            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20075        );
20076    }
20077
20078    #[test]
20079    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20080        let state = review_state(&["feat: the change"]);
20081        let source = landing_subject_source(&state);
20082        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20083        assert_eq!(
20084            land::merge_subject(landing_title(&state, old), &source),
20085            "feat: the change"
20086        );
20087        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20088        let task = state_with_summary("add retries", "");
20089        assert_eq!(landing_title(&task, old), old);
20090    }
20091
20092    #[test]
20093    fn pr_message_describes_the_change_not_the_task() {
20094        let state = state_with_summary(
20095            "今回やってほしいこと: results projector を直す",
20096            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20097        );
20098        let m = pr_message(&state, 'A');
20099        assert_eq!(m.title, "fix(web): batch the runs list reads");
20100        assert!(
20101            m.body.starts_with("## Summary\n\n- reads run.json once"),
20102            "{}",
20103            m.body
20104        );
20105        assert!(!m.body.contains("TITLE:"), "{}", m.body);
20106        let task_at = m.body.find("今回やってほしいこと").unwrap();
20107        let details_at = m.body.find("<details>").unwrap();
20108        assert!(
20109            details_at < task_at,
20110            "the task lives inside <details>: {}",
20111            m.body
20112        );
20113        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20114        assert!(m.body.contains("magi:candidate-a"));
20115    }
20116
20117    #[test]
20118    fn pr_message_falls_back_to_the_task_without_a_title_line() {
20119        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20120        let m = pr_message(&state, 'A');
20121        assert_eq!(m.title, "add retries");
20122        assert!(
20123            m.body.contains("## Summary\n\n- did some things"),
20124            "{}",
20125            m.body
20126        );
20127
20128        let none = RunState::new(
20129            PathBuf::from("/repo"),
20130            "main".to_owned(),
20131            "abc1234".to_owned(),
20132            "add retries".to_owned(),
20133            Config::default(),
20134        );
20135        let m = pr_message(&none, 'A');
20136        assert_eq!(m.title, "add retries");
20137        assert!(!m.body.contains("## Summary"), "{}", m.body);
20138    }
20139
20140    #[test]
20141    fn pr_message_refuses_the_candidate_commit_subject() {
20142        for bad in [
20143            "TITLE: magi: candidate A (uncommitted work)",
20144            "TITLE: chore: stuff (uncommitted work)",
20145            "TITLE:   ",
20146        ] {
20147            let state = state_with_summary("add retries", bad);
20148            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20149        }
20150    }
20151
20152    #[test]
20153    fn pr_message_bounds_a_very_long_task_and_title() {
20154        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20155        let state = state_with_summary(&long, "- nothing");
20156        let m = pr_message(&state, 'A');
20157        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20158        assert!(!m.title.contains('\n'));
20159
20160        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20161        let m = pr_message(&state, 'A');
20162        assert!(m.title.starts_with("feat: "));
20163        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20164        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20165    }
20166
20167    fn long_title_of(instruction: &str) -> String {
20168        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20169    }
20170
20171    #[test]
20172    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20173        let first = "Make the landing path keep a readable title for long tasks";
20174        let line = format!(
20175            "{first}. {}",
20176            "And then keep going with more words ".repeat(20)
20177        );
20178        let t = long_title_of(&line);
20179        assert_eq!(t, first);
20180        assert!(!t.starts_with("chore: land"));
20181    }
20182
20183    #[test]
20184    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20185        let line = "word ".repeat(200);
20186        let t = long_title_of(&line);
20187        assert!(t.ends_with("word..."), "{t}");
20188        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20189    }
20190
20191    #[test]
20192    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20193        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20194            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20195        }
20196    }
20197
20198    #[test]
20199    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20200        let mut state = state_with_summary("再試行を追加する", "- nothing");
20201        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20202        state.config.graph.github_language = "ja".into();
20203        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20204    }
20205
20206    #[test]
20207    fn pr_message_title_limit_is_exact() {
20208        let at = "a".repeat(PR_TITLE_MAX);
20209        assert_eq!(long_title_of(&at), at);
20210        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20211        assert!(over.ends_with("..."), "{over}");
20212        assert_eq!(over.chars().count(), PR_TITLE_MAX);
20213    }
20214
20215    #[test]
20216    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20217        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20218        let t = long_title_of(&line);
20219        assert!(t.ends_with("..."), "{t}");
20220        assert!(t.is_ascii(), "{t}");
20221    }
20222
20223    #[test]
20224    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20225        let line = format!(
20226            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20227            "plus more filler words ".repeat(20)
20228        );
20229        let t = long_title_of(&line);
20230        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20231    }
20232
20233    #[test]
20234    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20235        let line = format!("\n\n# {}", "title words ".repeat(40));
20236        let state = state_with_summary(&line, "- nothing");
20237        let m = pr_message(&state, 'A');
20238        assert!(m.title.starts_with("title words"), "{}", m.title);
20239        assert_eq!(
20240            land::merge_subject(&m.title, &landing_subject_source(&state)),
20241            m.title
20242        );
20243        // An operator's rename wins untouched.
20244        assert_eq!(
20245            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20246            "feat: renamed by hand"
20247        );
20248    }
20249
20250    #[tokio::test]
20251    async fn github_text_rewrite_is_bounded_and_falls_back() {
20252        crate::run::pin_test_home();
20253        for (reply, accepted) in [
20254            (
20255                "TITLE: fix: retries\nAdd retries for failed requests.",
20256                true,
20257            ),
20258            (
20259                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20260                false,
20261            ),
20262            (
20263                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20264                true,
20265            ),
20266            (
20267                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20268                true,
20269            ),
20270            (
20271                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20272                true,
20273            ),
20274            (
20275                "TITLE: fix: retries\n日本語の説明です。 token=secret",
20276                false,
20277            ),
20278            ("TITLE: fix: retries\n", false),
20279        ] {
20280            let dir = tempfile::tempdir().unwrap();
20281            let mut runner = runner_at(RunStatus::Gating);
20282            runner.state = state_with_summary(
20283                "add retries",
20284                "TITLE: fix: retries\n日本語の説明を書きます。",
20285            );
20286            runner.state.repo = dir.path().to_owned();
20287            runner.state.candidates[0].worktree = dir.path().to_owned();
20288            let mut author = spec("alpha");
20289            author.command = vec![
20290                "sh".into(),
20291                "-c".into(),
20292                "printf '%s' \"$REWRITE_REPLY\"".into(),
20293            ];
20294            author.env.insert("REWRITE_REPLY".into(), reply.into());
20295            runner.state.config.agents = vec![author];
20296            seed_public_summary(&mut runner);
20297            let winner = runner.state.candidates[0].clone();
20298            let first = runner
20299                .guarded_pr_message(&winner, None, true)
20300                .await
20301                .unwrap();
20302            let message = if accepted {
20303                first.expect("a passing rewrite never asks")
20304            } else {
20305                // Withheld: the run parks on a question and posts nothing yet.
20306                assert!(first.is_none());
20307                assert!(runner.state.parked);
20308                let store = ask::Questions::open();
20309                let asked: Vec<_> = store
20310                    .list()
20311                    .into_iter()
20312                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20313                    .collect();
20314                assert_eq!(asked.len(), 1);
20315                assert!(!asked[0].detail.contains("secret"));
20316                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20317                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20318                assert_eq!(
20319                    asked[0]
20320                        .choices
20321                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
20322                    rewrite.is_some()
20323                );
20324                // Silence: the question lapses and the neutral text is used,
20325                // without a second rewrite call or a second question.
20326                store
20327                    .update(&asked[0].id, |q| {
20328                        q.abandon("test");
20329                        Ok(())
20330                    })
20331                    .unwrap();
20332                let again = runner
20333                    .guarded_pr_message(&winner, None, true)
20334                    .await
20335                    .unwrap()
20336                    .expect("an abandoned question falls back");
20337                let n = store
20338                    .list()
20339                    .into_iter()
20340                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20341                    .count();
20342                assert_eq!(n, 1, "asked once per rejected text");
20343                again
20344            };
20345            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20346            assert!(!message.body.contains("secret"));
20347            assert!(!message.body.contains("prefix"));
20348            assert!(!message.title.contains("secret"));
20349            assert_eq!(
20350                message.body.contains("Add retries for failed requests."),
20351                accepted
20352            );
20353            assert_eq!(
20354                runner.state.seats["github-text-rewrite"].turns, 1,
20355                "only one rewrite invocation"
20356            );
20357            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20358            if accepted {
20359                assert!(runner.state.github_text.is_none());
20360                assert!(!runner.state.parked);
20361                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20362                assert!(message.body.contains("## Final verification"));
20363                assert!(message.body.contains("Evidence: final diff"));
20364                assert!(
20365                    message
20366                        .body
20367                        .contains(&format!("magi:run/{}", runner.state.id))
20368                );
20369            }
20370            if !accepted {
20371                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20372                assert!(
20373                    message
20374                        .body
20375                        .contains(&format!("magi:run/{}", runner.state.id))
20376                );
20377            }
20378        }
20379    }
20380
20381    fn scripted(id: &str, reply: &str) -> AgentSpec {
20382        let mut a = spec(id);
20383        a.command = vec![
20384            "sh".into(),
20385            "-c".into(),
20386            "printf '%s' \"$REWRITE_REPLY\"".into(),
20387        ];
20388        a.env.insert("REWRITE_REPLY".into(), reply.into());
20389        a
20390    }
20391
20392    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20393    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20394
20395    /// Gate tests inject generated public output, not internal reports.
20396    fn seed_public_summary(runner: &mut Runner) {
20397        let repo = &runner.state.repo;
20398        let g = |args: &[&str]| {
20399            let out = std::process::Command::new("git")
20400                .args(args)
20401                .current_dir(repo)
20402                .quiet()
20403                .output()
20404                .unwrap();
20405            assert!(out.status.success(), "git {args:?}");
20406            String::from_utf8_lossy(&out.stdout).trim().to_owned()
20407        };
20408        g(&["init", "-q", "-b", "main"]);
20409        g(&[
20410            "-c",
20411            "user.name=Test",
20412            "-c",
20413            "user.email=test@example.invalid",
20414            "commit",
20415            "-q",
20416            "--allow-empty",
20417            "-m",
20418            "feat: fixture",
20419        ]);
20420        let head = g(&["rev-parse", "HEAD"]);
20421        g(&["update-ref", "refs/remotes/origin/main", &head]);
20422        let evidence = crate::pr_outcome::summary_evidence(
20423            &runner.state,
20424            crate::pr_outcome::FinalHead::At(&head),
20425        );
20426        let generated = &runner.state.candidates[0].summary;
20427        runner.state.change_summary = Some(crate::run::ChangeSummary {
20428            base: head.clone(),
20429            head,
20430            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
20431            title: summary_title(generated).unwrap(),
20432            body: summary_without_title(generated),
20433        });
20434    }
20435
20436    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
20437        let mut runner = runner_at(RunStatus::Gating);
20438        runner.state = state_with_summary("add retries", JA_SUMMARY);
20439        if review_only {
20440            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
20441        }
20442        runner.state.repo = dir.to_owned();
20443        runner.state.candidates[0].worktree = dir.to_owned();
20444        runner.state.config.agents = agents;
20445        seed_public_summary(&mut runner);
20446        runner
20447    }
20448
20449    fn gate_questions(runner: &Runner) -> usize {
20450        ask::Questions::open()
20451            .list()
20452            .into_iter()
20453            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20454            .count()
20455    }
20456
20457    #[tokio::test]
20458    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
20459        crate::run::pin_test_home();
20460        let dir = tempfile::tempdir().unwrap();
20461        let mut runner = rewrite_runner(
20462            dir.path(),
20463            true,
20464            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20465        );
20466        runner.state.config.roles.language_judge =
20467            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
20468        let winner = runner.state.candidates[0].clone();
20469        let msg = runner
20470            .guarded_pr_message(&winner, None, true)
20471            .await
20472            .unwrap()
20473            .expect("an English rewrite posts without asking");
20474        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
20475        assert!(msg.body.contains("<details>kept</details>"));
20476        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
20477        assert_eq!(gate_questions(&runner), 0);
20478        assert!(runner.state.github_text.is_none());
20479        assert!(!runner.state.seats.contains_key("impl-A"));
20480        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20481    }
20482
20483    #[tokio::test]
20484    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
20485        crate::run::pin_test_home();
20486        let dir = tempfile::tempdir().unwrap();
20487        let mut runner = rewrite_runner(
20488            dir.path(),
20489            false,
20490            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20491        );
20492        runner.state.config.roles.language_judge =
20493            Some(crate::config::AgentChoice::One("beta".into()));
20494        runner.state.seats.insert(
20495            "impl-A".into(),
20496            SeatState::new("impl-A", "alpha", runner.state.seed),
20497        );
20498        let winner = runner.state.candidates[0].clone();
20499        let msg = runner
20500            .guarded_pr_message(&winner, None, true)
20501            .await
20502            .unwrap()
20503            .expect("rewritten");
20504        assert!(msg.body.contains("<details>kept</details>"));
20505        assert_eq!(runner.state.seats["impl-A"].turns, 0);
20506        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20507        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20508    }
20509
20510    #[tokio::test]
20511    async fn rewrite_reuses_a_resumable_implementer_session() {
20512        crate::run::pin_test_home();
20513        let dir = tempfile::tempdir().unwrap();
20514        let mut runner = rewrite_runner(
20515            dir.path(),
20516            false,
20517            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20518        );
20519        runner.state.config.roles.language_judge =
20520            Some(crate::config::AgentChoice::One("beta".into()));
20521        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20522        seat.turns = 1;
20523        runner.state.seats.insert("impl-A".into(), seat);
20524        let winner = runner.state.candidates[0].clone();
20525        runner
20526            .guarded_pr_message(&winner, None, true)
20527            .await
20528            .unwrap()
20529            .expect("rewritten");
20530        assert_eq!(runner.state.seats["impl-A"].turns, 2);
20531        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20532    }
20533
20534    #[tokio::test]
20535    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20536        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
20537        // session, was out of quota, and nothing else was asked.
20538        crate::run::pin_test_home();
20539        let dir = tempfile::tempdir().unwrap();
20540        let mut exhausted = spec("alpha");
20541        exhausted.command = vec!["false".into()];
20542        let mut runner = rewrite_runner(
20543            dir.path(),
20544            false,
20545            vec![exhausted, scripted("beta", EN_REWRITE)],
20546        );
20547        // The implementer's own agent is in the role too: it is not asked twice.
20548        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20549            "alpha".into(),
20550            "beta".into(),
20551        ]));
20552        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20553        seat.turns = 1;
20554        runner.state.seats.insert("impl-A".into(), seat);
20555        let winner = runner.state.candidates[0].clone();
20556        let msg = runner
20557            .guarded_pr_message(&winner, None, true)
20558            .await
20559            .unwrap()
20560            .expect("the role rewrites after the implementer fails");
20561        assert!(msg.body.contains("`src/net.rs`"));
20562        assert_eq!(
20563            runner.state.seats["impl-A"].turns, 2,
20564            "the implementer was tried once"
20565        );
20566        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20567        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20568    }
20569
20570    #[tokio::test]
20571    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20572        crate::run::pin_test_home();
20573        let dir = tempfile::tempdir().unwrap();
20574        let mut broken = spec("bad");
20575        broken.command = vec!["false".into()];
20576        let mut runner = rewrite_runner(
20577            dir.path(),
20578            true,
20579            vec![
20580                broken,
20581                scripted("noformat", "Add retries."),
20582                scripted("good", EN_REWRITE),
20583            ],
20584        );
20585        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20586            "bad".into(),
20587            "noformat".into(),
20588            "good".into(),
20589        ]));
20590        let winner = runner.state.candidates[0].clone();
20591        let msg = runner
20592            .guarded_pr_message(&winner, None, true)
20593            .await
20594            .unwrap()
20595            .expect("the third agent rewrites");
20596        assert!(msg.body.contains("`src/net.rs`"));
20597        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20598        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20599    }
20600
20601    #[tokio::test]
20602    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20603        crate::run::pin_test_home();
20604        // (agents, role): still Japanese; no TITLE; nothing resolvable.
20605        for (agents, role, saved) in [
20606            (
20607                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20608                Some("beta"),
20609                true,
20610            ),
20611            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20612            (Vec::new(), None, false),
20613        ] {
20614            let dir = tempfile::tempdir().unwrap();
20615            let mut runner = rewrite_runner(dir.path(), true, agents);
20616            runner.state.config.roles.language_judge =
20617                role.map(|r| crate::config::AgentChoice::One(r.into()));
20618            let winner = runner.state.candidates[0].clone();
20619            let first = runner
20620                .guarded_pr_message(&winner, None, true)
20621                .await
20622                .unwrap();
20623            assert!(first.is_none(), "withheld text parks the run");
20624            assert!(runner.state.parked);
20625            assert_eq!(gate_questions(&runner), 1);
20626            let g = runner.state.github_text.as_ref().unwrap();
20627            assert_eq!(g.rewrite.is_some(), saved);
20628            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20629        }
20630    }
20631
20632    #[tokio::test]
20633    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20634        crate::run::pin_test_home();
20635        let dir = tempfile::tempdir().unwrap();
20636        let mut runner = runner_at(RunStatus::Gating);
20637        runner.state =
20638            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20639        runner.state.repo = dir.path().to_owned();
20640        runner.state.candidates[0].worktree = dir.path().to_owned();
20641        let mut author = spec("alpha");
20642        author.command = vec![
20643            "sh".into(),
20644            "-c".into(),
20645            "printf '%s' \"$REWRITE_REPLY\"".into(),
20646        ];
20647        author.env.insert(
20648            "REWRITE_REPLY".into(),
20649            format!(
20650                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20651                "English sentence. ".repeat(600)
20652            ),
20653        );
20654        runner.state.config.agents = vec![author];
20655        seed_public_summary(&mut runner);
20656        let winner = runner.state.candidates[0].clone();
20657        assert!(
20658            runner
20659                .guarded_pr_message(&winner, None, true)
20660                .await
20661                .unwrap()
20662                .is_none()
20663        );
20664        let record = runner.state.github_text.clone().unwrap();
20665        let rewrite = record.rewrite.clone().unwrap();
20666        assert_eq!(rewrite.failed, ["body-language"]);
20667        assert!(
20668            rewrite
20669                .body
20670                .ends_with(&format!("magi:run/{}", runner.state.id))
20671        );
20672        let store = ask::Questions::open();
20673        let q = store.get(record.question.as_ref().unwrap()).unwrap();
20674        assert!(
20675            q.choices
20676                .contains(&crate::github_text::USE_REWRITE.to_owned())
20677        );
20678        for label in [
20679            "ORIGINAL",
20680            "FALLBACK",
20681            "REWRITE",
20682            "Rules still failed: body-language",
20683            "truncated",
20684        ] {
20685            assert!(q.detail.contains(label), "{label}");
20686        }
20687        let artifact = std::fs::read_to_string(crate::run::artifact_path(
20688            &runner.state,
20689            &crate::github_text::artifact_name(&record.fingerprint),
20690        ))
20691        .unwrap();
20692        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20693        // Restart from serialized state, then retry after a bad owner title.
20694        runner.state =
20695            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20696        store
20697            .update(&q.id, |q| {
20698                q.say("まだ日本語")?;
20699                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20700            })
20701            .unwrap();
20702        assert!(
20703            runner
20704                .guarded_pr_message(&winner, None, true)
20705                .await
20706                .unwrap()
20707                .is_none()
20708        );
20709        let retry = runner.state.github_text.as_ref().unwrap();
20710        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20711        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20712        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20713        assert!(
20714            q.posting_comparison
20715                .as_ref()
20716                .unwrap()
20717                .matches_detail(&q.detail)
20718        );
20719        store
20720            .update(&q.id, |q| {
20721                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20722            })
20723            .unwrap();
20724        let message = runner
20725            .guarded_pr_message(&winner, None, true)
20726            .await
20727            .unwrap()
20728            .unwrap();
20729        assert_eq!(message.title, rewrite.title);
20730        assert_eq!(message.body, rewrite.body);
20731        assert_eq!(
20732            runner.state.github_text.as_ref().unwrap().choice,
20733            Some(GithubTextChoice::Rewrite)
20734        );
20735        let again = runner
20736            .guarded_pr_message(&winner, None, true)
20737            .await
20738            .unwrap()
20739            .unwrap();
20740        assert_eq!(again.title, message.title);
20741        assert_eq!(again.body, message.body);
20742        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20743        assert!(
20744            !runner
20745                .state
20746                .events
20747                .iter()
20748                .any(|e| e.message.contains(&rewrite.title)
20749                    || e.message.contains("English sentence."))
20750        );
20751    }
20752
20753    #[tokio::test]
20754    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20755        crate::run::pin_test_home();
20756        let dir = tempfile::tempdir().unwrap();
20757        let mut runner = runner_at(RunStatus::Gating);
20758        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20759        runner.state.repo = dir.path().to_owned();
20760        runner.state.candidates[0].worktree = dir.path().to_owned();
20761        seed_public_summary(&mut runner);
20762        let winner = runner.state.candidates[0].clone();
20763        assert!(
20764            runner
20765                .guarded_pr_message(&winner, None, true)
20766                .await
20767                .unwrap()
20768                .is_none()
20769        );
20770        let record = runner.state.github_text.as_ref().unwrap();
20771        assert!(record.rewrite.is_none());
20772        // Records written before the additive field deserialize with no rewrite.
20773        let mut old = serde_json::to_value(record).unwrap();
20774        old.as_object_mut().unwrap().remove("rewrite");
20775        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20776        assert!(old.rewrite.is_none());
20777        let id = record.question.clone().unwrap();
20778        ask::Questions::open()
20779            .update(&id, |q| {
20780                q.choices.push(crate::github_text::USE_REWRITE.into());
20781                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20782            })
20783            .unwrap();
20784        let message = runner
20785            .guarded_pr_message(&winner, None, true)
20786            .await
20787            .unwrap()
20788            .unwrap();
20789        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20790        assert_eq!(
20791            runner.state.github_text.as_ref().unwrap().choice,
20792            Some(GithubTextChoice::Fallback)
20793        );
20794    }
20795
20796    #[tokio::test]
20797    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20798        crate::run::pin_test_home();
20799        let dir = tempfile::tempdir().unwrap();
20800        let mut runner = runner_at(RunStatus::Gating);
20801        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20802        runner.state.repo = dir.path().to_owned();
20803        runner.state.candidates[0].worktree = dir.path().to_owned();
20804        seed_public_summary(&mut runner);
20805        let winner = runner.state.candidates[0].clone();
20806        assert!(
20807            runner
20808                .guarded_pr_message(&winner, None, true)
20809                .await
20810                .unwrap()
20811                .is_none()
20812        );
20813        let store = ask::Questions::open();
20814        let find = |run: &str| {
20815            store
20816                .list()
20817                .into_iter()
20818                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20819                .unwrap()
20820        };
20821        let run = runner.state.id.clone();
20822        let q = find(&run);
20823        let fp = runner
20824            .state
20825            .github_text
20826            .as_ref()
20827            .unwrap()
20828            .fingerprint
20829            .clone();
20830        let name = crate::github_text::artifact_name(&fp);
20831        let path = crate::run::artifact_path(&runner.state, &name);
20832        assert!(q.detail.contains(&path.display().to_string()));
20833        assert!(q.detail.contains("title-language: title"));
20834        let saved = std::fs::read_to_string(&path).expect("artifact written");
20835        assert!(saved.contains("日本語のタイトル"));
20836        assert!(q.cwd.is_none());
20837        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20838        // A bad replacement is refused and asked about once more.
20839        let say = |id: &str, text: &str, choice: &str| {
20840            store
20841                .update(id, |q| {
20842                    q.thread.push(ask::Turn {
20843                        who: ask::Who::Operator,
20844                        body: text.to_owned(),
20845                        at: jiff::Timestamp::now(),
20846                        note: None,
20847                    });
20848                    q.answer(ask::Answer::Choice(choice.to_owned()))
20849                })
20850                .unwrap();
20851        };
20852        say(&q.id, "まだ日本語", "use my text");
20853        assert!(
20854            runner
20855                .guarded_pr_message(&winner, None, true)
20856                .await
20857                .unwrap()
20858                .is_none()
20859        );
20860        let second = find(&run);
20861        assert_ne!(second.id, q.id);
20862        assert!(second.detail.contains("did not pass"));
20863        // The retry still compares the original text; the rejected
20864        // replacement is named by category only.
20865        assert!(second.detail.contains("日本語のタイトル"));
20866        assert!(!second.detail.contains("まだ日本語"));
20867        assert!(second.detail.contains(&path.display().to_string()));
20868        say(&second.id, "fix: retry failed requests", "use my text");
20869        let message = runner
20870            .guarded_pr_message(&winner, None, true)
20871            .await
20872            .unwrap()
20873            .expect("a vetted replacement is posted");
20874        assert_eq!(message.title, "fix: retry failed requests");
20875        assert_eq!(
20876            runner
20877                .state
20878                .github_text
20879                .as_ref()
20880                .unwrap()
20881                .chosen_title
20882                .as_deref(),
20883            Some("fix: retry failed requests")
20884        );
20885        assert!(runner.state.github_text.as_ref().unwrap().resolved);
20886        // Resumed again: the saved decision is applied, nothing is re-asked.
20887        let again = runner
20888            .guarded_pr_message(&winner, None, true)
20889            .await
20890            .unwrap()
20891            .unwrap();
20892        assert_eq!(again.title, message.title);
20893        assert_eq!(
20894            store
20895                .list()
20896                .into_iter()
20897                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
20898                .count(),
20899            2
20900        );
20901    }
20902
20903    #[test]
20904    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
20905        // What magi itself writes stays English under any configured language,
20906        // so a future localisation of these headings fails here. (The agents'
20907        // own text is also checked by the posting gate.)
20908        let mut state = state_with_summary(
20909            "add retries",
20910            "TITLE: fix(web): batch reads\n- reads run.json once",
20911        );
20912        state.config.graph.language = "ja".to_owned();
20913        let m = pr_message(&state, 'A');
20914        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
20915        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
20916
20917        // The task is the operator's own text: it goes in untouched, and the
20918        // fallback title (no summary) may be in its language too.
20919        let task = "今回やってほしいこと: results projector を直す";
20920        let mut state = state_with_summary(task, "- no title line");
20921        state.config.graph.language = "ja".to_owned();
20922        let m = pr_message(&state, 'A');
20923        assert_eq!(
20924            m.title,
20925            format!("chore: land candidate A of run {}", state.id)
20926        );
20927        assert!(
20928            m.body.contains(&format!(
20929                "<summary>Original task</summary>\n\n{task}\n\n</details>"
20930            )),
20931            "{}",
20932            m.body
20933        );
20934    }
20935
20936    #[test]
20937    fn pr_message_scrubs_home_paths_and_addresses() {
20938        let state = state_with_summary(
20939            "fix it in /Users/someone/src/x",
20940            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
20941        );
20942        let m = pr_message(&state, 'A');
20943        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
20944            assert!(!m.body.contains(leak), "{}", m.body);
20945        }
20946        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
20947    }
20948
20949    #[test]
20950    fn pr_message_survives_a_task_that_closes_details() {
20951        let state = state_with_summary("a </details> b", "TITLE: fix: x");
20952        let m = pr_message(&state, 'A');
20953        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
20954    }
20955
20956    #[test]
20957    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
20958        let cmd = manual_merge_command(
20959            MergeStyle::Squash,
20960            Path::new("/repo"),
20961            "b",
20962            "fix: \"quoted\" $(x) `y`\n\nbody",
20963        );
20964        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
20965    }
20966
20967    #[test]
20968    fn manual_merge_command_matches_the_configured_style() {
20969        let repo = Path::new("/repo");
20970        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
20971
20972        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
20973        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
20974
20975        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
20976        assert_eq!(
20977            squash,
20978            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
20979             \"Merge magi run 0832 (candidate A)\""
20980        );
20981
20982        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
20983        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
20984    }
20985
20986    #[test]
20987    fn a_nudge_gets_a_quarter_of_the_budget() {
20988        // The judge and implement budgets magi ships with.
20989        assert_eq!(retry_budget(secs(1200), true), secs(300));
20990        assert_eq!(retry_budget(secs(3600), true), secs(900));
20991    }
20992
20993    #[test]
20994    fn a_resent_prompt_keeps_the_whole_budget() {
20995        // The seat kept no context, so the retry is the original job again and
20996        // shortening it would only guarantee a second failure.
20997        assert_eq!(retry_budget(secs(1200), false), secs(1200));
20998        assert_eq!(retry_budget(secs(60), false), secs(60));
20999    }
21000
21001    #[test]
21002    fn the_floor_never_exceeds_the_original_budget() {
21003        // A short configured timeout must not be *raised* by the floor: the
21004        // operator asked for a bound, and a retry may not outlast the attempt
21005        // it is retrying.
21006        assert_eq!(retry_budget(secs(60), true), secs(60));
21007        assert_eq!(retry_budget(secs(480), true), secs(120));
21008        assert_eq!(retry_budget(secs(0), true), secs(0));
21009    }
21010
21011    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21012        agent::CommandEvidence {
21013            id: "item1".to_owned(),
21014            description: "cargo test".to_owned(),
21015            exit_code,
21016            result_summary: String::new(),
21017            source: "codex".to_owned(),
21018        }
21019    }
21020
21021    #[test]
21022    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21023        // No evidence is not the same fact as unconfirmed evidence: a
21024        // backend with no adapter, or a reply that ran no commands at all,
21025        // must not be misread as carrying a dangling job.
21026        assert!(!has_unconfirmed_command(&[]));
21027    }
21028
21029    #[test]
21030    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21031        // Deliberately not a check on the exit code's *value*: a fixer
21032        // legitimately runs something that fails mid-iteration before it
21033        // succeeds, and that must never by itself reopen a valid report.
21034        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21035        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21036        assert!(!has_unconfirmed_command(&[
21037            evidence(Some(0)),
21038            evidence(Some(101))
21039        ]));
21040    }
21041
21042    #[test]
21043    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21044        assert!(has_unconfirmed_command(&[
21045            evidence(Some(0)),
21046            evidence(None)
21047        ]));
21048    }
21049
21050    #[test]
21051    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21052        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21053        assert_eq!(
21054            verified_noop_claim(true, &[], text).as_deref(),
21055            Some("already fixed by b32cfc4, on main.")
21056        );
21057    }
21058
21059    #[test]
21060    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21061        // A timeout or a bad exit code reads as the ordinary loss it is,
21062        // whatever the reply's own prose claims.
21063        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21064        assert!(verified_noop_claim(false, &[], text).is_none());
21065    }
21066
21067    #[test]
21068    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21069        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21070        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21071        // A confirmed command alongside the marker is fine.
21072        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21073    }
21074
21075    #[test]
21076    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21077        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21078    }
21079
21080    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
21081    /// pair, in order, labelled A, B, C, ...
21082    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21083        runner.state.candidates = shape
21084            .iter()
21085            .enumerate()
21086            .map(|(i, &(empty, verified))| Candidate {
21087                index: i,
21088                label: (b'A' + i as u8) as char,
21089                agent: "sonnet".to_owned(),
21090                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21091                worktree: PathBuf::from(format!("/wt/{i}")),
21092                summary: String::new(),
21093                stat: String::new(),
21094                files: 0,
21095                commits: 0,
21096                empty,
21097                failed: None,
21098                verified_noop: verified.map(str::to_owned),
21099                duration_ms: 0,
21100                folded: false,
21101            })
21102            .collect();
21103    }
21104
21105    #[test]
21106    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21107        ask_test_home();
21108        let mut runner = runner_at(RunStatus::Implementing);
21109        set_candidates(
21110            &mut runner,
21111            &[
21112                (true, Some("already on main at b32cfc4")),
21113                (true, Some("same fix, see the existing test")),
21114            ],
21115        );
21116
21117        runner
21118            .after_implement()
21119            .expect("a verified no-op is not an error");
21120
21121        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21122    }
21123
21124    #[test]
21125    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21126        ask_test_home();
21127        let mut runner = runner_at(RunStatus::Implementing);
21128        // Candidate A declares a verified no-op; candidate B simply wrote
21129        // nothing and said nothing about why. One candidate's claim is not
21130        // the whole run's agreement.
21131        set_candidates(
21132            &mut runner,
21133            &[(true, Some("already on main at b32cfc4")), (true, None)],
21134        );
21135
21136        let err = runner
21137            .after_implement()
21138            .expect_err("an unverified empty candidate must still fail the run");
21139
21140        assert!(
21141            err.to_string().contains("no candidate produced a change"),
21142            "{err}"
21143        );
21144        assert_eq!(runner.state.status, RunStatus::Failed);
21145    }
21146
21147    #[test]
21148    fn after_implement_still_fails_an_ordinary_all_empty_run() {
21149        ask_test_home();
21150        let mut runner = runner_at(RunStatus::Implementing);
21151        set_candidates(&mut runner, &[(true, None), (true, None)]);
21152
21153        let err = runner
21154            .after_implement()
21155            .expect_err("no candidate declared anything; this is an ordinary failure");
21156
21157        assert!(
21158            err.to_string().contains("no candidate produced a change"),
21159            "{err}"
21160        );
21161        assert_eq!(runner.state.status, RunStatus::Failed);
21162    }
21163}
21164
21165/// A graft rollback whose restore could not be confirmed (see
21166/// [`Runner::graft`]).
21167#[derive(Debug)]
21168struct GraftRestoreFailed(String);
21169
21170impl std::fmt::Display for GraftRestoreFailed {
21171    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21172        write!(
21173            f,
21174            "could not restore the winner to its pre-graft commit: {}",
21175            self.0
21176        )
21177    }
21178}
21179
21180impl std::error::Error for GraftRestoreFailed {}
21181
21182/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
21183/// not the agent: they never draw on an agent's own timeout, and running out of
21184/// them never reads as an agent failure.
21185#[cfg(test)]
21186mod resource_wait_tests {
21187    use super::*;
21188    use std::collections::BTreeMap;
21189
21190    fn sh(id: &str, script: &str) -> AgentSpec {
21191        AgentSpec {
21192            id: id.to_owned(),
21193            kind: crate::config::AgentKind::Command,
21194            model: None,
21195            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21196            extra_args: Vec::new(),
21197            env: BTreeMap::new(),
21198            prompt_delivery: None,
21199            max_turn_secs: None,
21200        }
21201    }
21202
21203    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21204        SeatJob {
21205            seat: SeatState::new("impl-A", &spec.id, 1),
21206            spec,
21207            cwd: dir.to_path_buf(),
21208            prompt: "do it".to_owned(),
21209            timeout,
21210            allow_write: true,
21211            sessions: false,
21212            artifacts: dir.join("art"),
21213            stem: "impl-A".to_owned(),
21214            handover: None,
21215        }
21216    }
21217
21218    fn state(resource_wait_secs: u64) -> RunState {
21219        let mut config = Config::default();
21220        config.graph.timeout_resource_wait = resource_wait_secs;
21221        RunState::new(
21222            PathBuf::from("/repo"),
21223            "main".to_owned(),
21224            "abc1234".to_owned(),
21225            "task".to_owned(),
21226            config,
21227        )
21228    }
21229
21230    fn hold(cache: &Path) -> crate::cache::Guard {
21231        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21232        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21233            crate::cache::AcquireOutcome::Acquired(g) => g,
21234            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21235        }
21236    }
21237
21238    async fn run(
21239        jobs: Vec<SeatJob>,
21240        sem: Arc<Semaphore>,
21241        cache: Option<&Path>,
21242        st: &mut RunState,
21243    ) -> Vec<(usize, SeatState, AgentOutcome)> {
21244        let prompts = Prompts::default();
21245        let ctx = WaveCtx {
21246            run: "r",
21247            node: "implement",
21248            prompts: &prompts,
21249            cache,
21250            round: None,
21251            carry_seats: false,
21252            pauses: None,
21253        };
21254        wave(jobs, sem, &ctx, st, 0).await
21255    }
21256
21257    #[tokio::test]
21258    async fn verification_cache_wait_preserves_the_command_budget() {
21259        crate::run::pin_test_home();
21260        let tmp = tempfile::tempdir().unwrap();
21261        let cache = tmp.path().join("target");
21262        let guard = hold(&cache);
21263        let release = tokio::spawn(async move {
21264            tokio::time::sleep(Duration::from_millis(100)).await;
21265            guard.release();
21266        });
21267        let mut st = state(10);
21268        let budget = Duration::from_millis(50);
21269        let wait = repository_wait(&st, None);
21270        let (outcomes, _) = with_cache_lease(
21271            &mut st,
21272            Some(&cache),
21273            "gate",
21274            "gate",
21275            tmp.path(),
21276            "abc1234",
21277            budget,
21278            "test gate",
21279            &wait,
21280            |_, available| async move {
21281                assert_eq!(
21282                    available, budget,
21283                    "contention must not shorten verification"
21284                );
21285                (Vec::new(), false, Vec::new())
21286            },
21287        )
21288        .await
21289        .unwrap();
21290        release.await.unwrap();
21291        assert!(outcomes.is_empty(), "{outcomes:?}");
21292        assert!(
21293            st.events
21294                .iter()
21295                .any(|event| event.message.contains("waiting for"))
21296        );
21297        assert!(st.resource_wait.is_none());
21298        // The verification body no longer owns the lease after returning.
21299        hold(&cache).release();
21300    }
21301
21302    #[tokio::test]
21303    async fn verification_cache_wait_expires_without_starting_commands() {
21304        crate::run::pin_test_home();
21305        let tmp = tempfile::tempdir().unwrap();
21306        let cache = tmp.path().join("target");
21307        let _held = hold(&cache);
21308        let mut st = state(0);
21309        let wait = repository_wait(&st, None);
21310        let (outcomes, _) = with_cache_lease(
21311            &mut st,
21312            Some(&cache),
21313            "gate",
21314            "gate",
21315            tmp.path(),
21316            "abc1234",
21317            Duration::from_secs(60),
21318            "test gate",
21319            &wait,
21320            |_, _| async { panic!("commands must not start without the cache lease") },
21321        )
21322        .await
21323        .unwrap();
21324        assert_eq!(outcomes.len(), 1);
21325        assert!(outcomes[0].resource_blocked);
21326        assert!(st.resource_wait.is_none());
21327        assert!(matches!(
21328            st.resource_block.unwrap().cause,
21329            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21330        ));
21331    }
21332
21333    #[tokio::test]
21334    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21335        crate::run::pin_test_home();
21336        let tmp = tempfile::tempdir().unwrap();
21337        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21338        let cache = tmp.path().join("cache");
21339        let mut state = state(21600);
21340        let wait = repository_wait(&state, None);
21341        let (outcomes, _) = with_cache_lease(
21342            &mut state,
21343            Some(&cache),
21344            "gate",
21345            "gate",
21346            tmp.path(),
21347            "head",
21348            Duration::from_secs(60),
21349            "gate",
21350            &wait,
21351            |_, _| async { panic!("freshness failure must not start verification") },
21352        )
21353        .await
21354        .unwrap();
21355        assert_eq!(outcomes.len(), 1);
21356        assert!(outcomes[0].resource_blocked);
21357        let block = state.resource_block.unwrap();
21358        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21359        assert!(!block.description().contains("timed out"));
21360    }
21361
21362    #[tokio::test]
21363    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21364        crate::run::pin_test_home();
21365        let tmp = tempfile::tempdir().unwrap();
21366        let cache = tmp.path().join("cache");
21367        let _held = hold(&cache);
21368        let mut state = state(21600);
21369        let pause = Pause::new();
21370        let interrupt = Pause::new();
21371        let cause = ParkCause::Operator;
21372        pause.park_for(cause.clone());
21373        let prompts = Prompts::default();
21374        let ctx = WaveCtx {
21375            run: "test",
21376            node: "fix",
21377            prompts: &prompts,
21378            cache: Some(&cache),
21379            round: None,
21380            carry_seats: false,
21381            pauses: Some((&pause, &interrupt)),
21382        };
21383        let result = wave(
21384            vec![job(
21385                sh("writer", "exit 99"),
21386                tmp.path(),
21387                Duration::from_secs(1),
21388            )],
21389            Arc::new(Semaphore::new(1)),
21390            &ctx,
21391            &mut state,
21392            0,
21393        )
21394        .await;
21395        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21396        assert_eq!(result[0].1.turns, 0);
21397        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21398        assert_eq!(
21399            state.resource_block.unwrap().cause,
21400            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21401        );
21402        assert!(state.active.is_empty());
21403    }
21404
21405    #[tokio::test]
21406    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21407        crate::run::pin_test_home();
21408        let tmp = tempfile::tempdir().unwrap();
21409        let cache = tmp.path().join("target-a");
21410        let guard = hold(&cache);
21411        // Held for longer than the agent's whole 1s budget.
21412        tokio::spawn(async move {
21413            tokio::time::sleep(Duration::from_millis(1500)).await;
21414            guard.release();
21415        });
21416        let mut st = state(60);
21417        let results = run(
21418            vec![job(
21419                sh("alpha", "echo ok"),
21420                tmp.path(),
21421                Duration::from_secs(1),
21422            )],
21423            Arc::new(Semaphore::new(1)),
21424            Some(&cache),
21425            &mut st,
21426        )
21427        .await;
21428        assert!(
21429            matches!(results[0].2, AgentOutcome::Ok(_)),
21430            "the wait must not be charged to the agent"
21431        );
21432        assert!(st.resource_block.is_none());
21433        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21434        assert!(st.resource_wait.is_none() && st.active.is_empty());
21435    }
21436
21437    #[tokio::test]
21438    #[cfg(unix)]
21439    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
21440        use std::os::fd::AsRawFd as _;
21441        crate::run::pin_test_home();
21442        let tmp = tempfile::tempdir().unwrap();
21443        crate::git::git(tmp.path(), &["init"]).await.unwrap();
21444        let common = tmp.path().join(".git");
21445        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
21446        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
21447        assert_eq!(
21448            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
21449            0
21450        );
21451        let mut st = state(1);
21452        st.config.coordination = crate::config::test_coordination();
21453        let mut reader = job(
21454            sh("reader", "echo must-not-run"),
21455            tmp.path(),
21456            Duration::from_secs(30),
21457        );
21458        reader.allow_write = false;
21459        let results = run(
21460            vec![reader.clone()],
21461            Arc::new(Semaphore::new(1)),
21462            None,
21463            &mut st,
21464        )
21465        .await;
21466        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21467        assert_eq!(results[0].1.turns, 0);
21468        assert!(FailClass::of(&results[0].2).is_none());
21469        assert_eq!(
21470            st.resource_block.as_ref().unwrap().kind,
21471            "repository coordination"
21472        );
21473        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21474        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21475        drop(external);
21476        let editor = crate::repo_lock::SeatLease::for_checkout(
21477            tmp.path(),
21478            true,
21479            crate::config::test_coordination().as_ref(),
21480        )
21481        .await
21482        .unwrap();
21483        let pause = Pause::new();
21484        let interrupt = Pause::new();
21485        let prompts = Prompts::default();
21486        let ctx = WaveCtx {
21487            run: "r",
21488            node: "review",
21489            prompts: &prompts,
21490            cache: None,
21491            round: None,
21492            carry_seats: false,
21493            pauses: Some((&pause, &interrupt)),
21494        };
21495        let (results, ()) = tokio::join!(
21496            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
21497            async {
21498                tokio::time::sleep(Duration::from_millis(50)).await;
21499                pause.park();
21500            }
21501        );
21502        assert!(
21503            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
21504        );
21505        assert_eq!(results[0].1.turns, 0);
21506        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21507        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21508        drop(editor);
21509    }
21510
21511    #[tokio::test]
21512    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
21513        crate::run::pin_test_home();
21514        let tmp = tempfile::tempdir().unwrap();
21515        let cache = tmp.path().join("target-b");
21516        let _held = hold(&cache);
21517        let mut st = state(1);
21518        let results = run(
21519            vec![job(
21520                sh("alpha", "echo ok"),
21521                tmp.path(),
21522                Duration::from_secs(1),
21523            )],
21524            Arc::new(Semaphore::new(1)),
21525            Some(&cache),
21526            &mut st,
21527        )
21528        .await;
21529        let (_, seat, out) = &results[0];
21530        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21531        assert!(wave_blocked(&results));
21532        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21533        assert_eq!(seat.turns, 0, "nothing was asked");
21534        assert!(
21535            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21536            "no agent was spawned"
21537        );
21538        assert!(FailClass::of(out).is_none(), "not a failure class");
21539        let block = st.resource_block.as_ref().expect("recorded for the resume");
21540        assert_eq!(block.kind, "shared build cache");
21541        assert!(matches!(
21542            block.cause,
21543            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21544        ));
21545        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21546        assert!(st.active.is_empty());
21547    }
21548
21549    #[tokio::test]
21550    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21551        crate::run::pin_test_home();
21552        let tmp = tempfile::tempdir().unwrap();
21553        let mut st = state(1);
21554        let results = run(
21555            vec![job(
21556                sh("alpha", "echo ok"),
21557                tmp.path(),
21558                Duration::from_secs(1),
21559            )],
21560            Arc::new(Semaphore::new(0)),
21561            None,
21562            &mut st,
21563        )
21564        .await;
21565        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21566        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21567        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21568    }
21569
21570    #[tokio::test]
21571    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21572        crate::run::pin_test_home();
21573        let tmp = tempfile::tempdir().unwrap();
21574        let mut st = state(60);
21575        let results = run(
21576            vec![job(
21577                sh("alpha", "sleep 30"),
21578                tmp.path(),
21579                Duration::from_secs(1),
21580            )],
21581            Arc::new(Semaphore::new(1)),
21582            None,
21583            &mut st,
21584        )
21585        .await;
21586        let out = &results[0].2;
21587        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21588        let class = FailClass::of(out).expect("a failure");
21589        assert_eq!(class, FailClass::Timeout);
21590        assert!(should_hand_over(None, &class), "existing handover policy");
21591        assert!(st.resource_block.is_none());
21592    }
21593}