Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54    Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58    ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62/// How much verification output is kept and fed back to the fixer.
63const OUTPUT_TAIL: usize = 8_000;
64
65/// Bytes of a failing command's output kept in an event, so the reason a run
66/// stopped is readable from the report without opening `run.json`.
67const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
70/// command's pid before releasing the build cache's lease.
71const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
74/// command's pid to actually exit before giving up and releasing anyway.
75///
76/// A timeout means the process was asked to die (`kill_on_drop`,
77/// `start_kill`), not that it already has — on Windows in particular that can
78/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
79/// the instant the command returns would let the very next acquirer (this
80/// run's own next round, another run's verification, the janitor's prune)
81/// start touching the same directory while it might still be writing to it,
82/// so this polls the actual pid — real confirmation, not a fixed guess —
83/// until it is gone or this ceiling is reached. It is still not full
84/// process-tree reaping: a grandchild the timed-out process spawned and that
85/// outlives it independently is invisible to a pid check, and continuing to
86/// observe and collect *that* stays a different piece of work with its own
87/// owner. Set generously because the common case returns early the moment
88/// the pid is confirmed gone, not because every timeout pays this in full.
89const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91/// Consecutive review rounds with no tree progress (see
92/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
93/// instead of spending the rest of the round budget.
94///
95/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
96/// legitimately finds nothing left to change (its previous round's fix already
97/// covered it, and this round's reviewers re-raised only nits) looks the same
98/// as one that is spinning, for exactly one round. Two in a row is where the
99/// two stop being distinguishable, and a review round on this workload has
100/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
101/// third attempt at a tree that has not moved twice running is pure cost.
102/// This does not touch `review_rounds` itself, which stays the operator's
103/// call.
104pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
107/// a base that moved before giving up and leaving the run `Blocked` for a
108/// person.
109///
110/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
111/// that keeps moving faster than a run can catch it is not something more
112/// rebasing fixes, it is a person's call. Not the same *number as*
113/// `land_rounds` - this budget is spent before a pull request exists, land's
114/// after - but bounded for the identical reason, so it uses the same
115/// default. Counted across both call sites in [`Runner::finish_after_tally`]
116/// (once before review, once before the gate), because either one finding
117/// the base still moving is the same signal.
118const BASE_SYNC_ROUNDS: usize = 4;
119
120/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
121/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
122/// reply held no [`FixReport`].
123///
124/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
125/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
126/// "I'll pause here until the `cargo make check` background run reports
127/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
128/// No `FixReport` was ever collected from that seat, and the run moved on to
129/// the next review round regardless.
130///
131/// Bounded independently of `review_rounds` and `graph.retries`: this
132/// recovers one seat's missing report mid-round, not a new round of review or
133/// an ordinary parse retry, and must not itself become the unbounded wait the
134/// rest of this module exists to avoid.
135const MAX_FIX_CONTINUATIONS: usize = 2;
136
137/// One queued agent invocation.
138///
139/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
140/// CLI hung up on its own stream is asked again from the same job rather than
141/// rebuilt from scratch. See [`Runner::resume_undelivered`].
142#[derive(Clone)]
143struct SeatJob {
144    spec: AgentSpec,
145    seat: SeatState,
146    cwd: PathBuf,
147    prompt: String,
148    timeout: Duration,
149    allow_write: bool,
150    sessions: bool,
151    artifacts: PathBuf,
152    stem: String,
153    /// The prompt for a seat that has been handed to another roster agent
154    /// (a fresh session): everything the original seat would have
155    /// remembered. `None` when `prompt` already carries it, as the first
156    /// ranking and the implement prompt do. Never a resume-style prompt.
157    handover: Option<String>,
158}
159
160/// How the graph reads one agent invocation.
161///
162/// Quota is split out from an ordinary failure on purpose: a rate-limited call
163/// is known to fail again if retried now, so the retry loop must not spend an
164/// attempt on it. `Dropped` is split out for the opposite reason: unlike
165/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
166/// error JSON, never the agent's answer — a caller that matched only
167/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
168/// left to read that JSON as if it were usable output. `resume_undelivered`
169/// is the only caller that acts on it; everywhere else it is reported like an
170/// ordinary failure.
171enum AgentOutcome {
172    /// A usable output.
173    Ok(AgentOutput),
174    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
175    Quota(AgentOutput),
176    /// The CLI hung up on its own stream after billed work. See
177    /// [`agent::AgentOutput::work_undelivered`].
178    Dropped(AgentOutput),
179    /// Any other failure: a timeout, a bad exit code, an empty reply.
180    Failed(String),
181    /// A resource prevented admission or interrupted execution. Not an agent
182    /// failure: never advance a fallback chain or spend an attempt on it.
183    /// Post-spawn interruptions carry a typed error into wave, which records
184    /// implementation preservation before returning this outcome. This outcome
185    /// alone is not proof that no source changes were made.
186    /// [`RunState::resource_block`] says why.
187    ResourceBlocked(String),
188}
189
190/// A request to park the run at its next node boundary.
191///
192/// Cloning is how the request travels: the loop keeps one handle and hands a
193/// clone to each [`Runner`], and every clone points at the same flag. There
194/// is no channel because there is nothing to send - the only message is
195/// "park", it is idempotent, and a flag cannot be missed by a receiver that
196/// was not listening yet.
197///
198/// The boundary is what makes this cheap. Every node writes the run's state
199/// before the next one starts, and every node skips what is already recorded:
200/// `prep` returns early once candidates exist, `implement` asks only the seats
201/// with nothing on disk, `judge` returns early once judgements exist. So a
202/// parked run resumes into exactly the node it stopped before, and no agent
203/// work is thrown away. Killing the process mid-node, by contrast, loses
204/// whatever the seats in flight had not yet written - which for an implement
205/// wave is an hour of paid work.
206///
207/// A [`Runner`] watches two independent handles of this type - see
208/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
209/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
210/// clone covering the whole daemon's lifetime and is never asked to un-park,
211/// which is correct exactly because nothing is dispatched after it fires.
212/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
213/// that parks for an interrupted task must go on to run other tasks
214/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
215/// reusing the daemon-wide one.
216#[derive(Debug, Clone, Default)]
217pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
218
219impl Pause {
220    /// A pause nobody has asked for yet.
221    #[must_use]
222    pub fn new() -> Self {
223        Self::default()
224    }
225
226    /// Ask the run to park at its next node boundary. Idempotent.
227    pub fn park(&self) {
228        self.0.store(true, Ordering::SeqCst);
229    }
230
231    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
232    /// fold into the run's own `park` event - so an operator reading the run
233    /// later knows this was a deliberate interrupt rather than a shutdown or
234    /// a binary swap. The first reason recorded wins; a park already in
235    /// flight is not relabelled by a second, unrelated request.
236    pub fn park_because(&self, reason: impl Into<String>) {
237        self.park_for(ParkCause::Interrupt {
238            reason: reason.into(),
239        });
240    }
241
242    /// [`Pause::park`], recording the structured cause. The first cause
243    /// recorded wins, like [`Pause::park_because`].
244    pub fn park_for(&self, cause: ParkCause) {
245        let mut reason_guard = self
246            .1
247            .lock()
248            .unwrap_or_else(std::sync::PoisonError::into_inner);
249        if reason_guard.is_none() {
250            *reason_guard = Some(cause);
251        }
252        drop(reason_guard);
253        self.park();
254    }
255
256    /// Has a park been asked for?
257    #[must_use]
258    pub fn parked(&self) -> bool {
259        self.0.load(Ordering::SeqCst)
260    }
261
262    /// Why the park was asked for, when the caller used [`Pause::park_because`].
263    #[must_use]
264    pub fn reason(&self) -> Option<String> {
265        self.cause().map(|c| c.label())
266    }
267
268    /// The structured cause, when one was recorded.
269    #[must_use]
270    pub fn cause(&self) -> Option<ParkCause> {
271        self.1
272            .lock()
273            .unwrap_or_else(std::sync::PoisonError::into_inner)
274            .clone()
275    }
276}
277
278/// Drives one run.
279pub struct Runner {
280    /// Run state; public so the CLI can report on it.
281    pub state: RunState,
282    roles: ResolvedRoles,
283    sem: Arc<Semaphore>,
284    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
285    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
286    /// this is never the same handle as `interrupt`.
287    pause: Pause,
288    /// Set when `magi serve`'s interrupt scheduler wants this specific run
289    /// parked at its next node boundary, to let a task marked
290    /// [`crate::queue::Task::interrupt`] run alone before this one carries
291    /// on. Unlike `pause`, a fresh, unshared handle per run - see
292    /// [`Runner::watch_interrupt`].
293    interrupt: Pause,
294}
295
296/// Where the branch's own commits start: its merge base with the base branch
297/// as the remote has it now, else with the recorded `base_commit`. A branch
298/// rebased onto a base that moved past `base_commit` would otherwise count
299/// the base's commits as its own under `base_commit..branch`.
300async fn review_base(
301    repo: &Path,
302    remote: &str,
303    base_branch: &str,
304    base_commit: &str,
305    branch: &str,
306) -> String {
307    review_base_checked(repo, remote, base_branch, base_commit, branch)
308        .await
309        .0
310}
311
312/// [`review_base`] plus whether the base was read from a freshly fetched
313/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
314/// never older than `base_commit`'s view of the base), but the answer is then
315/// not trusted to rewrite a pull request's title.
316async fn review_base_checked(
317    repo: &Path,
318    remote: &str,
319    base_branch: &str,
320    base_commit: &str,
321    branch: &str,
322) -> (String, bool) {
323    let tracking = format!("{remote}/{base_branch}");
324    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
325    if git::rev_exists(repo, &tracking).await
326        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
327        && !mb.is_empty()
328    {
329        return (mb, fresh);
330    }
331    let mb = git::merge_base(repo, base_commit, branch)
332        .await
333        .ok()
334        .filter(|mb| !mb.is_empty())
335        .unwrap_or_else(|| base_commit.to_owned());
336    (mb, false)
337}
338
339/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
340/// when git cannot say or finds nothing: a stale list is better than a wrong
341/// or empty one.
342pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
343    if !is_review_run(state) {
344        return;
345    }
346    let base = review_base(
347        &state.repo,
348        &state.config.merge.remote,
349        &state.base_branch,
350        &state.base_commit,
351        branch,
352    )
353    .await;
354    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
355        && !subjects.is_empty()
356        && state.reviewed_commits.as_ref() != Some(&subjects)
357    {
358        state.reviewed_commits = Some(subjects);
359        state.save().ok();
360    }
361}
362
363/// Subjects of the base's commits between the recorded start and the branch's
364/// merge base: what a stale `base_commit..branch` would have mistaken for the
365/// branch's own work.
366async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
367    let (base, trusted) = review_base_checked(
368        &state.repo,
369        &state.config.merge.remote,
370        &state.base_branch,
371        &state.base_commit,
372        branch,
373    )
374    .await;
375    if !trusted {
376        return None;
377    }
378    git::subjects(&state.repo, &state.base_commit, &base)
379        .await
380        .ok()
381}
382
383/// May an adopted pull request's title be replaced with `computed`? Only when
384/// it is empty, magi's own shape, or a base commit's subject that leaked in;
385/// a title a person wrote stays. Never when `computed` is itself a leak.
386pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
387    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
388    if is_leak(computed) {
389        return false;
390    }
391    let cur = current.trim();
392    cur.is_empty()
393        || cur.starts_with(REVIEW_PROMPT_OPENING)
394        || cur.starts_with("chore: land candidate")
395        || cur.starts_with("magi: candidate")
396        || is_leak(cur)
397}
398
399/// The commit a run branches from: the base branch as the remote has it.
400///
401/// Two failures this replaces. A run used to branch off `HEAD` and so refused
402/// to start on a dirty tree, which made `magi serve` decline every task for as
403/// long as the operator had work in progress - most of the time. Branching off
404/// the *local* base branch fixed that and introduced a worse one: `land` merges
405/// the winner on GitHub, nothing updates the local ref, and the next run
406/// branches off a base missing everything the previous runs landed. Two tasks
407/// in a row from a phone would have had the second silently re-implementing
408/// against stale code and opening a pull request that reverted the first.
409///
410/// Only refs move here - no checkout, no local branch, no merge - so it is safe
411/// with uncommitted work in the tree. A machine with no network still starts:
412/// the fetch may fail and the local tip is used with a warning, because
413/// refusing to run offline is a worse failure than running against a base the
414/// operator can see for themselves.
415///
416/// One function, called by both entry points. Two answers to "where does a run
417/// branch from" is the kind of drift nobody notices until a diff is wrong.
418/// Bring the local `branch` in line with `<remote>/<branch>` before a review
419/// checks it out.
420///
421/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
422/// anything other than plain `git push` from this checkout (a jj colocated
423/// workspace, another clone) moves only the remote-tracking ref - so the local
424/// one can be a stale placeholder. It moves only when local is behind the remote or is an
425/// empty placeholder that diverged from it; unpushed local work is kept, and a real
426/// divergence is refused rather than guessed at.
427async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
428    let tracking = format!("{remote}/{branch}");
429    let fetched = git::fetch(repo, remote, branch).await;
430    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
431    let local_exists = git::branch_exists(repo, branch).await?;
432    if !fresh {
433        if !local_exists {
434            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
435        }
436        tracing::warn!(
437            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
438        );
439        return Ok(());
440    }
441    let remote_sha = git::rev_parse(repo, &tracking).await?;
442    if !local_exists {
443        git::git(repo, &["branch", branch, &tracking]).await?;
444        return Ok(());
445    }
446    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
447    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
448        return Ok(());
449    }
450    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
451        // Diverged. `reconcile` settles it only when it can prove nothing is
452        // lost: a local tip that is the remote's change rebased is pushed over
453        // it (lease pinned to the tip read here), a tip whose every extra
454        // commit is empty is a placeholder the remote's work replaced, and
455        // anything else is two different changes - a question for a person.
456        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
457            .await?
458        {
459            crate::reconcile::Reconciliation::Pushed => {
460                tracing::warn!(
461                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
462                    short(&local_sha),
463                    short(&remote_sha)
464                );
465                return Ok(());
466            }
467            crate::reconcile::Reconciliation::Placeholder => {}
468            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
469        }
470    }
471    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
472    if !out.ok() {
473        bail!(
474            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
475            short(&local_sha),
476            short(&remote_sha),
477            out.stderr
478        );
479    }
480    tracing::warn!(
481        "local `{branch}` was stale: fast-forwarded {} -> {}",
482        short(&local_sha),
483        short(&remote_sha)
484    );
485    Ok(())
486}
487
488async fn resolve_base(
489    repo: &Path,
490    base_branch: &str,
491    remote: &str,
492    coordination: Option<&crate::config::Coordination>,
493) -> Result<crate::run::FetchedBase> {
494    let _lease = crate::repo_lock::SeatLease::operation_with(
495        repo,
496        coordination,
497        &crate::repo_lock::WaitPolicy::current(),
498    )
499    .await?;
500    let tracking = format!("{remote}/{base_branch}");
501    let fetched = git::fetch(repo, remote, base_branch).await;
502    if let Ok(out) = &fetched
503        && out.ok()
504        && git::rev_exists(repo, &tracking).await
505    {
506        return Ok(crate::run::FetchedBase {
507            remote: remote.to_owned(),
508            branch: base_branch.to_owned(),
509            commit: git::rev_parse(repo, &tracking).await?,
510            fetched_at: jiff::Timestamp::now(),
511        });
512    }
513    let why = match &fetched {
514        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
515        Ok(_) => format!("{remote} has no {base_branch}"),
516        Err(e) => e.to_string(),
517    };
518    // No fallback to the local branch: it may be behind, and branching off an
519    // old commit is the stale-checkout bug this check exists to prevent.
520    bail!(
521        "cannot read {tracking} ({why}); refusing to branch off the local \
522         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
523         base / remote in magi.toml. This is launcher preparation: check the \
524         launcher's Git administration access, network and repository coordination \
525         before retrying; an unattended seat cannot approve those permissions"
526    )
527}
528
529/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
530/// on an early return or a panic.
531///
532/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
533/// manual `magi fix` invocations against the same run are otherwise
534/// invisible to each other and would race to remove and recreate the same
535/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
536/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
537/// which is only ever reclaimed later, out of band, by
538/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
539/// `magi fix` invocation is not necessarily running under either of those, so
540/// nothing would ever sweep a lock a killed or crashed process left behind.
541/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
542/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
543/// lease use: an unreadable or unparsable pid, or a liveness query the
544/// platform cannot answer, reads as alive and the lock is left in place.
545struct FixClaim {
546    path: PathBuf,
547}
548
549impl FixClaim {
550    fn acquire(dir: &Path) -> Result<Self> {
551        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
552        let path = dir.join("fix.lock");
553        let retry = crate::lock_file::Retry::new(None);
554        match Self::create(&path, retry) {
555            Ok(claim) => Ok(claim),
556            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
557                if Self::reclaim_if_dead(&path) {
558                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
559                } else {
560                    bail!(
561                        "another repair or `magi fix` is already running for this run ({} exists)",
562                        path.display()
563                    )
564                }
565            }
566            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
567        }
568    }
569
570    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
571        let mut f = retry.create(path)?;
572        use std::io::Write as _;
573        // Read back by `reclaim_if_dead` on a later, stuck invocation.
574        writeln!(f, "{}", std::process::id())?;
575        Ok(Self {
576            path: path.to_owned(),
577        })
578    }
579
580    /// True if the lock named a process confirmed dead, in which case it was
581    /// also removed. Never true on an unreadable file, an unparsable pid, or
582    /// a liveness query the platform cannot answer — see this type's own doc.
583    fn reclaim_if_dead(path: &Path) -> bool {
584        let dead = std::fs::read_to_string(path)
585            .ok()
586            .and_then(|body| body.trim().parse::<u32>().ok())
587            .is_some_and(|pid| !crate::proc::pid_alive(pid));
588        dead && std::fs::remove_file(path).is_ok()
589    }
590}
591
592impl Drop for FixClaim {
593    fn drop(&mut self) {
594        let _ = std::fs::remove_file(&self.path);
595    }
596}
597
598/// Holds the mutable runner as well as its durable lease: cancellation must
599/// clear both copies of ownership, including state a caller may save later.
600struct DriverExecution<'a> {
601    runner: &'a mut Runner,
602    driver: crate::driver::DriverGuard,
603}
604
605impl Drop for DriverExecution<'_> {
606    fn drop(&mut self) {
607        self.driver.release_local(&mut self.runner.state);
608    }
609}
610
611impl Runner {
612    /// Start a fresh run against `repo`.
613    pub async fn start(
614        repo: &Path,
615        instruction: String,
616        config: Config,
617        origin: Origin,
618    ) -> Result<Self> {
619        Self::start_naming(repo, instruction, "", config, origin).await
620    }
621
622    /// [`Runner::start`] for a queued task: `also_scan` (the task's title) is
623    /// searched for branch and commit references along with the instruction,
624    /// since a task may name the work it is about only in its title.
625    pub async fn start_naming(
626        repo: &Path,
627        instruction: String,
628        also_scan: &str,
629        config: Config,
630        origin: Origin,
631    ) -> Result<Self> {
632        let repo = git::toplevel(repo).await?;
633        let missing = agent::missing_programs(&config.agents);
634        if !missing.is_empty() {
635            bail!(
636                "these agent programs are not on PATH: {}. Fix the roster in \
637                 magi.toml or install them.",
638                missing.join(", ")
639            );
640        }
641        let base_branch =
642            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643                .await?;
644        let fetched_base = resolve_base(
645            &repo,
646            &base_branch,
647            &config.merge.remote,
648            config.coordination.as_ref(),
649        )
650        .await?;
651        let base_commit = fetched_base.commit.clone();
652        let roles = config.resolve_roles()?;
653        let max_parallel = config.graph.max_parallel.max(1);
654        // A task that points at work already in the repository starts from
655        // it; what the repository says about each reference is recorded.
656        let seeds = refs::resolve(
657            &repo,
658            &base_commit,
659            &config.merge.remote,
660            &format!("{also_scan}\n{instruction}"),
661        )
662        .await;
663        refs::plan(&repo, &seeds).await?;
664        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
665        // Recorded before the first save, so a crash right after minting
666        // cannot leave a run with no origin. Legibility only: nothing reads it
667        // to decide anything.
668        state.fetched_base = Some(fetched_base);
669        state.origin = Some(origin);
670        for seed in &seeds {
671            state.event(
672                "seed",
673                refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
674            );
675        }
676        state.seeds = seeds;
677        state.event("start", format!("run {} created", state.id));
678        state.save()?;
679        Ok(Self {
680            state,
681            roles,
682            sem: Arc::new(Semaphore::new(max_parallel)),
683            pause: Pause::new(),
684            interrupt: Pause::new(),
685        })
686    }
687
688    /// Open a review-only run against work that already exists on `branch`.
689    ///
690    /// The expensive half of the graph is the implement wave — measured at
691    /// 111 and 134 internal tool-loop turns on this repository, against a
692    /// handful for a judge or a reviewer. The cheap half is worth running on
693    /// hand-written work too, and there was no way to reach it.
694    ///
695    /// No new state and no schema change are needed: a run with **one** viable
696    /// candidate and a tally already decided degrades `execute` to exactly
697    /// review → gate → merge, because `judge` skips a single-candidate field,
698    /// `deliberate` has fewer than two first choices to reconcile, `vote`
699    /// returns early, `tally` is already present and `fold_losers` has no
700    /// losers. Resuming such a run therefore does the right thing as well.
701    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
702        Self::review_taking_over(repo, branch, config, None, origin).await
703    }
704
705    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
706    /// at the same task still has `branch` checked out, its worktree is
707    /// released first if that is safe (see [`crate::handover`]), and the
708    /// review refuses with the reason if it is not. `None` is a hand-run
709    /// review: it has no earlier attempts, so only a worktree of a dead run
710    /// magi recorded itself can be released.
711    pub async fn review_taking_over(
712        repo: &Path,
713        branch: &str,
714        config: Config,
715        takeover: Option<crate::handover::Takeover>,
716        origin: Origin,
717    ) -> Result<Self> {
718        let repo = git::toplevel(repo).await?;
719        let missing = agent::missing_programs(&config.agents);
720        if !missing.is_empty() {
721            bail!(
722                "these agent programs are not on PATH: {}. Fix the roster in \
723                 magi.toml or install them.",
724                missing.join(", ")
725            );
726        }
727        let base_branch =
728            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
729                .await?;
730        if base_branch == branch {
731            bail!("`{branch}` is the base branch; there is nothing to review against");
732        }
733        let fetched_base = resolve_base(
734            &repo,
735            &base_branch,
736            &config.merge.remote,
737            config.coordination.as_ref(),
738        )
739        .await?;
740        let base_commit = fetched_base.commit.clone();
741
742        let roles = config.resolve_roles()?;
743        let max_parallel = config.graph.max_parallel.max(1);
744        let mut state = RunState::new(
745            repo.clone(),
746            base_branch,
747            base_commit.clone(),
748            String::new(),
749            config,
750        );
751        state.fetched_base = Some(fetched_base);
752        state.origin = Some(origin);
753
754        // Released before anything else touches the branch: a stale local
755        // branch is moved with `git branch -f`, which git refuses while an
756        // earlier attempt's worktree still has it checked out. Everything
757        // after this point that can fail puts the old run back.
758        // A hand-run review has no task, hence no earlier attempts, but a
759        // worktree of a dead run magi made may still be released.
760        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
761            earlier: Vec::new(),
762            home: crate::run::home(),
763            choice: None,
764            task_instruction: None,
765        });
766        let wait = repository_wait(&state, None);
767        let coordination_config = state.config.coordination.clone();
768        let coordination =
769            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
770                .await?;
771        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
772        if let Some(released) = &released {
773            state.event(
774                "release",
775                format!(
776                    "took `{branch}` over from run {}: its worktree was released: {}",
777                    crate::run::short_of(&released.old_id),
778                    released.audit
779                ),
780            );
781        }
782        // The owner's answer to an earlier divergence question is applied
783        // here: after the release (git will not move a checked-out branch)
784        // and before the sync that would otherwise ask again.
785        if let Some(choice) = takeover.choice.as_ref()
786            && let Err(e) =
787                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
788                    .await
789        {
790            if let Some(released) = &released {
791                released.restore(&repo, branch).await;
792            }
793            return Err(e.context("applying the owner's answer about the diverged branch"));
794        }
795        // An earlier attempt of this task that published the task head, and
796        // whose work is the branch under review, hands that head over: the
797        // review run updates the same pull request. Without one (a legacy
798        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
799        // own head and its own open pull request is adopted as before.
800        for id in takeover.earlier.iter().rev() {
801            if let Ok(old) = RunState::load_under(id, &takeover.home)
802                && let Some(head) = old.pr_head.clone()
803                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
804            {
805                state.pr_head = Some(head);
806                break;
807            }
808        }
809        drop(coordination);
810        let opened = Self::open_review(
811            &repo,
812            branch,
813            state,
814            roles,
815            max_parallel,
816            base_commit,
817            &takeover,
818        )
819        .await;
820        if opened.is_err()
821            && let Some(released) = &released
822        {
823            let _coordination = crate::repo_lock::SeatLease::operation_with(
824                &repo,
825                coordination_config.as_ref(),
826                &wait,
827            )
828            .await?;
829            released.restore(&repo, branch).await;
830        }
831        opened
832    }
833
834    /// Recover exactly the operator-selected public implementation, without touching old sources.
835    pub async fn recover_pr(
836        repo: &Path,
837        task: &str,
838        selection: crate::pr_recovery::Selection,
839        mut config: Config,
840    ) -> Result<Self> {
841        selection.configure(&mut config)?;
842        selection.check_task(task, &selection.instruction)?;
843        let base_branch =
844            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
845                .await?;
846        let fetched_base = resolve_base(
847            repo,
848            &base_branch,
849            &config.merge.remote,
850            config.coordination.as_ref(),
851        )
852        .await?;
853        let base_commit = fetched_base.commit.clone();
854        selection
855            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
856            .await?;
857        Box::pin(selection.verify_source(repo)).await?;
858        if let Some(proof) = &selection.source_proof {
859            anyhow::ensure!(
860                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
861                "source rebase base is outside the fetched landing base"
862            );
863        }
864        let roles = config.resolve_roles()?;
865        let max_parallel = config.graph.max_parallel.max(1);
866        let mut state = RunState::new(
867            repo.to_path_buf(),
868            base_branch,
869            base_commit.clone(),
870            selection.instruction.clone(),
871            config,
872        );
873        state.fetched_base = Some(fetched_base);
874        state.origin = Some(crate::run::Origin::queue(task));
875        state.pr_head = Some(selection.branch.clone());
876        state.pr_recovery = Some(selection.clone());
877        crate::pr_head::check_predecessors(&state, &selection.branch).await?;
878        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
879    }
880
881    async fn open_selected(
882        repo: &Path,
883        state: RunState,
884        roles: ResolvedRoles,
885        max_parallel: usize,
886        base_commit: String,
887        selection: crate::pr_recovery::Selection,
888    ) -> Result<Self> {
889        let mut state = state;
890        let branch = state.branch_for('A');
891        let wait = repository_wait(&state, None);
892        let coordination = coordinate_operation(&mut state, repo, &wait).await?;
893        git::git(repo, &["branch", &branch, selection.source()]).await?;
894        drop(coordination);
895        state.event(
896            "recovery",
897            format!(
898                "selected implementation {} for {} (public lease {})",
899                selection.source(),
900                selection.url,
901                selection.head
902            ),
903        );
904        let takeover = crate::handover::Takeover {
905            earlier: Vec::new(),
906            home: crate::run::home(),
907            choice: None,
908            task_instruction: Some(selection.instruction.clone()),
909        };
910        state.save()?;
911        let id = state.id.clone();
912        let opened = Self::open_review(
913            repo,
914            &branch,
915            state,
916            roles,
917            max_parallel,
918            base_commit,
919            &takeover,
920        )
921        .await;
922        match opened {
923            Ok(runner) => Ok(runner),
924            Err(e) => {
925                if let Ok(mut partial) = RunState::load(&id) {
926                    partial.status = RunStatus::Blocked;
927                    partial.driver_exited = true;
928                    partial.event(
929                        "recovery",
930                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
931                    );
932                    partial.save()?;
933                }
934                Err(crate::pr_recovery::StartupFailed {
935                    run: id,
936                    cause: format!("{e:#}"),
937                }
938                .into())
939            }
940        }
941    }
942
943    /// What the newest earlier attempt that reviewed `branch` leaves for this
944    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
945    /// no earlier attempts. Reads the earlier record only: ownership of its
946    /// worktree and branch is [`crate::handover`]'s business.
947    async fn inherit(
948        repo: &Path,
949        branch: &str,
950        base_commit: &str,
951        takeover: &crate::handover::Takeover,
952    ) -> Option<crate::recovery::RecoveryRecord> {
953        use crate::recovery::{Relation, assess};
954        let newest = takeover.earlier.last()?;
955        let mut chosen = None;
956        for id in takeover.earlier.iter().rev() {
957            if let Ok(old) = RunState::load_under(id, &takeover.home)
958                && !old.reviews.is_empty()
959                && (old.winner().is_some_and(|w| w.branch == branch)
960                    || old.pr_head.as_deref() == Some(branch))
961            {
962                chosen = Some(old);
963                break;
964            }
965        }
966        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
967            .await
968            .unwrap_or_default();
969        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
970            Some(last) if last.head == tip => Relation::Same,
971            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
972            _ => Relation::Diverged,
973        };
974        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
975        Some(assess(from, chosen.as_ref(), base_commit, relation))
976    }
977
978    /// The half of [`Runner::review_taking_over`] that can fail after an
979    /// earlier attempt's worktree was released.
980    async fn open_review(
981        repo: &Path,
982        branch: &str,
983        mut state: RunState,
984        roles: ResolvedRoles,
985        max_parallel: usize,
986        base_commit: String,
987        takeover: &crate::handover::Takeover,
988    ) -> Result<Self> {
989        let wait = repository_wait(&state, None);
990        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
991        if state.pr_recovery.is_none() {
992            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
993        }
994        // The commit subjects are the closest thing to a task statement that
995        // existing work carries, and the reviewers are told as much.
996        let start = review_base(
997            repo,
998            &state.config.merge.remote,
999            &state.base_branch,
1000            &base_commit,
1001            branch,
1002        )
1003        .await;
1004        let log = git::log_oneline(repo, &start, branch)
1005            .await
1006            .unwrap_or_default();
1007        let log_text = if log.trim().is_empty() {
1008            "(no commit messages)"
1009        } else {
1010            log.trim()
1011        };
1012        let mut instruction = format!(
1013            "Review the work already on branch `{branch}`. There is no task \
1014             statement: what the change claims to do is whatever its commits \
1015             say.\n\n{log_text}"
1016        );
1017        // A retry of a task whose earlier attempt left reviews behind carries
1018        // that attempt's evidence, and the task's own statement with it: the
1019        // completion criteria are the task's, not whatever the commits claim.
1020        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1021            if let Some(task) = takeover
1022                .task_instruction
1023                .as_deref()
1024                .filter(|t| !t.trim().is_empty())
1025                && rec.reviewed_head.is_some()
1026            {
1027                instruction = format!(
1028                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1029                    task.trim()
1030                );
1031                rec.criteria_inherited = true;
1032            }
1033            state.event("recovery", rec.summary());
1034            state.recovery = Some(rec);
1035        }
1036        if let Some(selected) = &state.pr_recovery {
1037            instruction = selected.instruction.clone();
1038        }
1039        state.task_instruction = takeover.task_instruction.clone();
1040        state.instruction = instruction;
1041        state.reviewed_commits = Some(
1042            git::subjects(repo, &start, branch)
1043                .await
1044                .unwrap_or_default(),
1045        );
1046
1047        // An attached worktree, so the fixer's commits land on the branch under
1048        // review rather than on a detached head nobody will look at again.
1049        let worktree = state.worktree_root().join("under-review");
1050        if let Some(parent) = worktree.parent() {
1051            tokio::fs::create_dir_all(parent).await.ok();
1052        }
1053        let path = worktree.to_string_lossy().to_string();
1054        git::git(repo, &["worktree", "add", &path, branch])
1055            .await
1056            .with_context(|| {
1057                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1058            })?;
1059
1060        let setup = match ensure_setup_config(&mut state, repo).await {
1061            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1062            Err(e) => Err(e),
1063        };
1064        if let Err(e) = setup {
1065            git::worktree_remove_clean(repo, &worktree).await.ok();
1066            return Err(e);
1067        }
1068        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1069            .await
1070            .unwrap_or(0);
1071        if commits == 0 {
1072            git::worktree_remove_clean(repo, &worktree).await.ok();
1073            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1074        }
1075        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1076            .await
1077            .map(|f| f.len())
1078            .unwrap_or(0);
1079        if files == 0
1080            && let (Ok(head_tree), Ok(base_tree)) = (
1081                git::tree_of(&worktree, "HEAD").await,
1082                git::tree_of(&worktree, &base_commit).await,
1083            )
1084            && head_tree == base_tree
1085        {
1086            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1087            git::worktree_remove_clean(repo, &worktree).await.ok();
1088            bail!(
1089                "`{branch}` at {} has a tree identical to base {}; this usually means \
1090                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1091                 against `{}/{branch}`) rather than an empty change",
1092                short(&head),
1093                short(&base_commit),
1094                state.config.merge.remote
1095            );
1096        }
1097        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1098            .await
1099            .unwrap_or_default();
1100
1101        state.candidates.push(Candidate {
1102            index: 0,
1103            label: 'A',
1104            // Not an agent id on purpose: nothing in the roster wrote this, and
1105            // the stats tables must not credit anyone with a win for it.
1106            agent: EXISTING_BRANCH.to_owned(),
1107            branch: branch.to_owned(),
1108            worktree,
1109            summary: String::new(),
1110            stat,
1111            files,
1112            commits,
1113            empty: false,
1114            failed: None,
1115            verified_noop: None,
1116            duration_ms: 0,
1117            folded: false,
1118        });
1119        state.tally = Some(Tally {
1120            first_choice: BTreeMap::from([('A', 0)]),
1121            borda: BTreeMap::new(),
1122            winner: 'A',
1123            rankings: 0,
1124            unanimous_initial: false,
1125            deliberated: false,
1126            changed_votes: 0,
1127            unanimous_final: false,
1128            tie_break: None,
1129            // No panel sat, so no quorum applies. Zero judges is the correct
1130            // number for work that never competed, and must not be reported as
1131            // a collapsed panel.
1132            judges: 0,
1133            present: 0,
1134            quorum: 0,
1135            met_quorum: true,
1136            uncontested: Some("review-only run: nothing competed".to_owned()),
1137        });
1138        state.status = RunStatus::Reviewing;
1139        state.prep_complete = Some(true);
1140        state.event(
1141            "start",
1142            format!(
1143                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1144                state.id
1145            ),
1146        );
1147        state.save()?;
1148        Ok(Self {
1149            state,
1150            roles,
1151            sem: Arc::new(Semaphore::new(max_parallel)),
1152            pause: Pause::new(),
1153            interrupt: Pause::new(),
1154        })
1155    }
1156
1157    /// Reopen an existing run.
1158    pub fn resume(id: &str) -> Result<Self> {
1159        let state = RunState::load(id)?;
1160        crate::pr_head::ensure_owner(&state)?;
1161        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1162            bail!(
1163                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1164            );
1165        }
1166        if let Some(to) = &state.released_to {
1167            bail!(
1168                "run {} cannot be resumed: its worktree was released to run {}",
1169                state.short(),
1170                crate::run::short_of(to)
1171            );
1172        }
1173        let roles = state.config.resolve_roles()?;
1174        if let Some(id) = &state.config.graph.selected_implementer
1175            && state.candidates.iter().any(|c| c.agent != *id)
1176        {
1177            bail!(
1178                "replacement implementer pin does not match the saved candidates; task must stay held"
1179            );
1180        }
1181        let max_parallel = state.config.graph.max_parallel.max(1);
1182        Ok(Self {
1183            state,
1184            roles,
1185            sem: Arc::new(Semaphore::new(max_parallel)),
1186            pause: Pause::new(),
1187            interrupt: Pause::new(),
1188        })
1189    }
1190
1191    /// Walk the graph to a terminal state, skipping nodes already recorded.
1192    ///
1193    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1194    /// the phone - ends here, so this is the one place a run that ended
1195    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1196    /// notification centre. Best-effort: see [`crate::notices::raise`].
1197    pub async fn execute(&mut self) -> Result<()> {
1198        if let Some(selection) = &self.state.pr_recovery
1199            && (selection.source_proof.is_some() || selection.source_run.is_some())
1200        {
1201            Box::pin(selection.verify_source(&self.state.repo)).await?;
1202        }
1203        // Queue claims cover daemon work; this also serializes manual drivers
1204        // consuming a repair answer, including the new review and decision.
1205        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1206            || self.state.land_approval.is_some()
1207            || self.state.merge_decision.is_some()
1208            || !self.state.merge_repairs.is_empty()
1209            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1210        {
1211            let claim = FixClaim::acquire(&self.state.dir())?;
1212            let disk = RunState::load(&self.state.id)?;
1213            let checkpoint = |s: &RunState| {
1214                (
1215                    s.merge_repairs.len(),
1216                    s.merge_repairs
1217                        .last()
1218                        .map(|r| (r.question.clone(), r.phase)),
1219                    s.land_approval
1220                        .as_ref()
1221                        .map(|a| (a.question.clone(), a.head.clone())),
1222                )
1223            };
1224            if checkpoint(&disk) != checkpoint(&self.state) {
1225                bail!(
1226                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1227                );
1228            }
1229            Some(claim)
1230        } else {
1231            None
1232        };
1233        let mut execution = self.claim_driver()?;
1234        let runner = &mut *execution.runner;
1235        let wait = runner.repository_wait();
1236        // Keep the graph's large future off the callers' stack, including
1237        // standalone task attempts running on a default-sized test thread.
1238        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1239        let result = match result {
1240            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1241                record_repository_block(&mut runner.state, &e);
1242                runner.park_here()?;
1243                Ok(())
1244            }
1245            other => other,
1246        };
1247        execution.driver.finish(&mut runner.state)?;
1248        let ended = if result.is_err() {
1249            Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1250        } else {
1251            crate::notices::run_ended(&runner.state)
1252        };
1253        if let Some(notice) = ended {
1254            crate::notices::raise(notice);
1255        }
1256        result
1257    }
1258
1259    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1260        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1261    }
1262
1263    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1264        let wait = self.repository_wait();
1265        coordinate_operation(&mut self.state, cwd, &wait).await
1266    }
1267
1268    /// Validate ownership and persist the driver claim in one critical section.
1269    fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1270        let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1271            Ok(owner) => owner,
1272            Err(error) => {
1273                let detail = format!("worktree maintenance or ownership update: {error:#}");
1274                // Admission did not own run.json: retain this resource outcome
1275                // in memory for queue settlement without overwriting a concurrent
1276                // retirement or publication transaction.
1277                self.state.resource_block = Some(crate::run::ResourceBlock {
1278                    node: "start".into(),
1279                    kind: "worktree maintenance".into(),
1280                    detail: detail.clone(),
1281                    at: Timestamp::now(),
1282                });
1283                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1284            }
1285        };
1286        let disk = RunState::load(&self.state.id)?;
1287        crate::pr_head::ensure_owner(&disk)?;
1288        if disk.liveness(false) == crate::run::Liveness::Live {
1289            bail!("run already has a live driver; wait for its execution to end");
1290        }
1291        if let Some(to) = &disk.released_to {
1292            bail!(
1293                "run {} cannot continue: its worktree was released to run {}",
1294                self.state.short(),
1295                crate::run::short_of(to)
1296            );
1297        }
1298        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1299        let mut execution = DriverExecution {
1300            runner: self,
1301            driver,
1302        };
1303        // Declare after the execution guard: on any claim error this short
1304        // lock must drop before that guard reloads and releases ownership.
1305        let _owner = owner;
1306        let state = &mut execution.runner.state;
1307        // Moving again, so it is no longer parked. Set before the walk rather
1308        // than in `resume`, so every way of re-entering the graph clears it
1309        // and a card cannot claim a run is waiting to be resumed while the
1310        // agents are already working.
1311        state.parked = false;
1312        state.resource_wait = None;
1313        state.resource_block = None;
1314        // Any seat this state still lists as answering belongs to whatever
1315        // process last drove this run — this one included, if it crashed
1316        // mid-wave. Cleared and flushed immediately, before anything else
1317        // runs, so a resume can never show a seat as live when nothing is
1318        // asking it anything yet; the node that actually dispatches the next
1319        // wave repopulates it.
1320        state.clear_active();
1321        // Recorded in the same spot, and flushed together with the clear
1322        // above: this is the pid a reader checks (`RunState::liveness`) when
1323        // no daemon claim exists to answer "is a process still driving this
1324        // run" — a plain `magi run` / `magi review` typed into a terminal
1325        // claims nothing there. Always overwritten, never only-if-absent, so
1326        // a resumed run's stale pid from a previous, possibly-dead process
1327        // can never survive into this one's own report. Unlike
1328        // `clear_active`, this changes on every single `execute()` call, so
1329        // the save below is now unconditional rather than only-if-cleared.
1330        //
1331        // `driver_started_at` is recorded in the same breath, from this same
1332        // pid, so `liveness` can tell a live pid that is genuinely still us
1333        // apart from one the OS has since handed to an unrelated process —
1334        // see that field's own doc for why the pid alone is not enough.
1335        let pid = std::process::id();
1336        state.driver_pid = Some(pid);
1337        state.driver_started_at = crate::proc::process_started_at(pid);
1338        state.driver_exited = false;
1339        state.begin_pass();
1340        execution.driver.bind(state);
1341        state.save()?;
1342        execution.driver.record(state)?;
1343        Ok(execution)
1344    }
1345
1346    async fn execute_graph(&mut self) -> Result<()> {
1347        self.state.configure_recovery_resume()?;
1348        // Preservation may invalidate an old empty classification. Resolve it
1349        // before any resume shortcut can reuse a verdict or publish a branch.
1350        self.persist_candidates().await?;
1351        if self
1352            .state
1353            .merge_decision
1354            .as_ref()
1355            .is_some_and(|decision| !decision.closed)
1356            && self.state.status != RunStatus::Landing
1357        {
1358            if crate::merge_decision::resumable_hold(&self.state) {
1359                self.state.merge = Some(MergeOutcome {
1360                    mode: MergeMode::Pr,
1361                    ok: false,
1362                    detail: self
1363                        .state
1364                        .merge_decision
1365                        .as_ref()
1366                        .unwrap()
1367                        .target
1368                        .pr
1369                        .clone(),
1370                    empty: false,
1371                });
1372                self.run_land().await?;
1373                self.settle_questions();
1374                return Ok(());
1375            }
1376            bail!(
1377                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1378            );
1379        }
1380        if self.state.config.graph.selected_implementer.is_some()
1381            && self.state.status == RunStatus::Blocked
1382            && self.state.tally.is_none()
1383            && self.state.candidate_preservation.is_empty()
1384            && self.state.interrupted_implementations.is_empty()
1385        {
1386            bail!(
1387                "selected implementation is held; explicitly release the owning task to retry it"
1388            );
1389        }
1390
1391        if self.state.merge_repairs.last().is_some_and(|r| {
1392            matches!(
1393                r.phase,
1394                crate::run::MergeRepairPhase::Requested
1395                    | crate::run::MergeRepairPhase::Fixing
1396                    | crate::run::MergeRepairPhase::Reviewing
1397            )
1398        }) {
1399            if self.repair_merge_review().await? {
1400                self.run_land().await?;
1401            }
1402            self.settle_questions();
1403            return Ok(());
1404        }
1405        // Publication already passed review and gate. Resume only that node:
1406        // syncing to a newer base here could change the verified commit.
1407        if self.state.pr_publication_pending {
1408            self.merge().await?;
1409            if self.resource_halt()? {
1410                return Ok(());
1411            }
1412            self.state.save()?;
1413            self.settle_questions();
1414            return Ok(());
1415        }
1416        // A run that already lost its quorum never resumes into the verdict
1417        // machinery: `deliberate` and `vote` would otherwise clobber the
1418        // stalled marker back to Voting and the run would keep going past a
1419        // verdict that is no longer trustworthy. Everything already recorded is
1420        // kept, so the run stays resumable (or foldable) for a human to pick up.
1421        //
1422        // On --resume the run gets one chance to repair itself: the seats a
1423        // rate limit took out are re-asked. If their quota has since reset and
1424        // the quorum is restored, the run picks up and finishes; otherwise it
1425        // stays stale and still-resumable for a later retry. If it does not
1426        // recover, the returned status stays `Stalled` and nothing was
1427        // clobbered (the recovery only mutates entries for the lost seats).
1428        if self.state.status == RunStatus::Stalled {
1429            if self.recover_stall().await? {
1430                self.finish_after_tally().await?;
1431            } else {
1432                // Still below quorum: persist the marker and stay resumable.
1433                self.state.save()?;
1434            }
1435            return Ok(());
1436        }
1437        // A run parked inside `land` - watching CI, mid fix-round, or
1438        // waiting on the owner's merge approval - resumes directly into it,
1439        // never back through `prep`. Everything before `merge` already
1440        // concluded; that is the only way `status` reaches `Landing` in the
1441        // first place. Re-walking `review_loop` first would also be actively
1442        // wrong: its own status recomputation (see its doc) treats any
1443        // clean round as reason to set `status` to `Gating`, which would
1444        // clobber this marker before `merge` ever ran, and this run would
1445        // never find its way back into `land` at all.
1446        if self.state.status == RunStatus::Landing
1447            || crate::handover::landing_checkpoint(&self.state)
1448        {
1449            self.state.land_sync_pending = true;
1450            self.state.save()?;
1451            self.run_land().await?;
1452            // `run_land` may have settled the run right here - CI came back
1453            // green and the PR merged, say - without ever passing back
1454            // through `merge`'s own trailing call. Whatever it left `status`
1455            // as is what this has to read.
1456            self.settle_questions();
1457            return Ok(());
1458        }
1459        // A run parked at `merge` on the owner's word about a withheld PR text
1460        // resumes straight into `merge`. Re-walking the tail would sync to the
1461        // base again (possibly rebasing, possibly spending the fixer) after the
1462        // gate had passed, and then push a tree nobody reviewed or gated.
1463        if self.state.github_text.is_some()
1464            && self.state.merge.is_none()
1465            && self.state.status == RunStatus::Gating
1466        {
1467            self.merge().await?;
1468            if self.resource_halt()? {
1469                return Ok(());
1470            }
1471            self.state.save()?;
1472            return Ok(());
1473        }
1474        if let Err(e) = self.prep().await {
1475            if e.is::<crate::repo_lock::ResourceBlocked>() {
1476                return Err(e);
1477            }
1478            self.state.status = RunStatus::Blocked;
1479            self.state
1480                .prep_failure
1481                .get_or_insert_with(|| format!("{e:#}"));
1482            self.state
1483                .event("prep", format!("preparation stopped: {e:#}"));
1484            self.state.save()?;
1485            return Err(e);
1486        }
1487        if self.park_here()? {
1488            return Ok(());
1489        }
1490        self.advise().await?;
1491        if self.park_here()? {
1492            return Ok(());
1493        }
1494        self.implement().await?;
1495        if self.park_here()? {
1496            return Ok(());
1497        }
1498        // `after_implement` already saved the state and settled any open
1499        // questions when it set this; nothing later in the graph has
1500        // anything to judge.
1501        if self.state.status == RunStatus::VerifiedNoop {
1502            return Ok(());
1503        }
1504        self.judge().await?;
1505        if self.park_here()? {
1506            return Ok(());
1507        }
1508        self.deliberate().await?;
1509        if self.park_here()? {
1510            return Ok(());
1511        }
1512        self.vote().await?;
1513        if self.park_here()? {
1514            return Ok(());
1515        }
1516        self.tally()?;
1517        // A verdict that lost its quorum is not trustworthy: do not review,
1518        // gate, or merge on it. Everything already done is kept, so the run
1519        // stays resumable (or foldable); the human can replace the agent that
1520        // ran out of quota and pick it up.
1521        if self.state.status == RunStatus::Stalled {
1522            // Persist the stalled marker now — the normal end-of-execute save
1523            // below is below this early return, and without it a resumed run
1524            // would reload a pre-tally status and keep going.
1525            self.state.save()?;
1526            return Ok(());
1527        }
1528        self.finish_after_tally().await?;
1529        Ok(())
1530    }
1531
1532    /// A wave could not start its agents because a shared resource stayed busy
1533    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1534    /// resumable. Unstarted seats have no result; completed peers retain
1535    /// their saved candidate state.
1536    fn stop_for_resource(&mut self) -> Result<()> {
1537        // A wait abandoned because the run is parking stays a park.
1538        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1539        {
1540            self.state.status = RunStatus::Blocked;
1541        }
1542        self.state.save()
1543    }
1544
1545    /// Stop walking the graph if a node ended on a resource block.
1546    fn resource_halt(&mut self) -> Result<bool> {
1547        if self.state.resource_block.is_none() {
1548            return Ok(false);
1549        }
1550        self.stop_for_resource()?;
1551        Ok(true)
1552    }
1553
1554    /// Park here if asked to, recording it in the run's own timeline.
1555    ///
1556    /// Returns whether the caller should stop walking the graph. The state is
1557    /// saved either way by the node that just finished; this adds the event so
1558    /// the operator's card says why a run that is neither finished nor moving
1559    /// is sitting where it is.
1560    fn park_here(&mut self) -> Result<bool> {
1561        let parking = self.pause.parked() || self.interrupt.parked();
1562        if !parking && self.state.resource_block.is_some() {
1563            self.stop_for_resource()?;
1564            return Ok(true);
1565        }
1566        // Either handle asking is enough - see `Pause`'s own doc for why
1567        // they are never the same one. `interrupt` is checked second so a
1568        // reason it carries is preferred in the message below over a plain
1569        // shutdown park racing it at the same boundary.
1570        if !self.pause.parked() && !self.interrupt.parked() {
1571            return Ok(false);
1572        }
1573        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1574        let why = match cause.as_ref().map(ParkCause::label) {
1575            Some(reason) => format!(
1576                "parked after `{}` ({reason}) — resume to carry on from here",
1577                self.state.status.as_str()
1578            ),
1579            None => format!(
1580                "parked after `{}` — resume to carry on from here",
1581                self.state.status.as_str()
1582            ),
1583        };
1584        self.state.event("park", why);
1585        self.state
1586            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1587        self.state.parked = true;
1588        self.state.save()?;
1589        Ok(true)
1590    }
1591
1592    /// Only a new explicit operator approval may retry failed preparation.
1593    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1594        if self.state.prep_failure.is_none() {
1595            return Ok(());
1596        }
1597        let failed_at = self
1598            .state
1599            .events
1600            .iter()
1601            .rev()
1602            .find(|event| event.node == "prep")
1603            .map_or(self.state.updated_at, |event| event.at);
1604        anyhow::ensure!(
1605            approval_at > failed_at,
1606            "preparation retry requires a new explicit release after the failure"
1607        );
1608        self.state.prep_failure = None;
1609        self.state.prep_complete = Some(false);
1610        self.state
1611            .event("prep-retry", "operator approved retry of preparation");
1612        Ok(())
1613    }
1614
1615    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1616    pub fn on_pause(&mut self, pause: Pause) {
1617        self.pause = pause;
1618    }
1619
1620    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1621    /// scheduler asking this one run - and no other - to park so a task
1622    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1623    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1624    /// handle.
1625    pub fn watch_interrupt(&mut self, pause: Pause) {
1626        self.interrupt = pause;
1627    }
1628
1629    /// Abandon this run's own open questions, once `status` has actually
1630    /// settled rather than merely paused.
1631    ///
1632    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1633    /// either back up with the candidates, the review round and the seat
1634    /// sessions already on disk, so a question an implementer asked mid-round
1635    /// may still get a real answer read by a real resume. Only the statuses
1636    /// `resumable` excludes are actually final: the run merged, it reached
1637    /// `Ready` with nothing left to do, it failed outright with no
1638    /// established point to continue from, or every candidate agreed, with
1639    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1640    /// every one of those the seat that asked is gone for good, exactly like
1641    /// the run being deleted under `magi run rm` - so the same cleanup
1642    /// applies, worded for what actually happened instead of "the run was
1643    /// deleted".
1644    ///
1645    /// Best-effort and silent on success: called from every place `status`
1646    /// can land on one of those three, including ones a resumed run revisits,
1647    /// so it must cost nothing when there was nothing open to begin with.
1648    fn settle_questions(&mut self) {
1649        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1650            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1651        }
1652    }
1653
1654    /// The tail of the graph after a trustworthy tally: fold losers, review,
1655    /// gate, merge, and persist.
1656    async fn finish_after_tally(&mut self) -> Result<()> {
1657        // Before the losers are folded: their branches are what the graft
1658        // reads from.
1659        self.graft().await?;
1660        if self.resource_halt()? {
1661            return Ok(());
1662        }
1663        self.fold_losers().await?;
1664        // Before review starts, and again right before the gate: a run's
1665        // review rounds can themselves take long enough for the base to move
1666        // a second time, and the gate is the one node whose "green" gets
1667        // acted on.
1668        self.sync_to_base().await?;
1669        if self.resource_halt()? {
1670            return Ok(());
1671        }
1672        if self.state.status == RunStatus::AlreadyInBase {
1673            return Ok(());
1674        }
1675        self.review_loop().await?;
1676        if self.resource_halt()? {
1677            return Ok(());
1678        }
1679        self.sync_to_base().await?;
1680        if self.resource_halt()? {
1681            return Ok(());
1682        }
1683        if self.state.status == RunStatus::AlreadyInBase {
1684            return Ok(());
1685        }
1686        self.gate().await?;
1687        self.merge().await?;
1688        if self.resource_halt()? {
1689            return Ok(());
1690        }
1691        self.state.save()?;
1692        Ok(())
1693    }
1694
1695    // ---------------------------------------------------------------- prep
1696
1697    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1698    /// run with the step and its output in the event; it is never swallowed.
1699    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1700    /// resume starts over instead of finding candidates and missing seats.
1701    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1702        let result = match ensure_setup_config(&mut self.state, repo).await {
1703            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1704            Err(e) => Err(e),
1705        };
1706        if let Err(e) = result {
1707            git::worktree_remove_clean(repo, wt).await.ok();
1708            if in_prep {
1709                self.rollback_prep(repo).await;
1710            }
1711            self.state.status = RunStatus::Blocked;
1712            self.state.event(
1713                "setup",
1714                format!("worktree setup failed in {}: {e:#}", wt.display()),
1715            );
1716            self.state.save()?;
1717            return Err(e);
1718        }
1719        Ok(())
1720    }
1721
1722    /// Undo everything `prep` created, so a resume re-runs it whole.
1723    async fn rollback_prep(&mut self, repo: &Path) {
1724        let root = self.state.worktree_root();
1725        for c in std::mem::take(&mut self.state.candidates) {
1726            if matches!(
1727                git::worktree_remove_clean(repo, &c.worktree).await,
1728                Ok(true)
1729            ) {
1730                git::branch_delete(repo, &c.branch).await.ok();
1731            } else {
1732                self.state.candidates.push(c);
1733            }
1734        }
1735        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1736        for j in 1..=seats {
1737            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1738                .await
1739                .ok();
1740        }
1741        for k in 1..=self.state.config.graph.advisors {
1742            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1743                .await
1744                .ok();
1745        }
1746    }
1747
1748    async fn prep(&mut self) -> Result<()> {
1749        if let Some(reason) = &self.state.prep_failure {
1750            bail!(
1751                "candidate preparation previously failed: {reason}; correct the source before retrying"
1752            );
1753        }
1754        if self
1755            .state
1756            .prep_complete
1757            .unwrap_or(!self.state.candidates.is_empty())
1758        {
1759            return Ok(());
1760        }
1761        let repo = self.state.repo.clone();
1762        // Includes rollback, config, refs, checkout creation and setup. No
1763        // repository mutation may precede the real shared lock acquisition.
1764        let _coordination = self.coordinate(&repo).await?;
1765        self.rollback_prep(&repo).await;
1766        if !self.state.candidates.is_empty() {
1767            bail!(
1768                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1769            );
1770        }
1771        self.state.prep_complete = Some(false);
1772        self.state.status = RunStatus::Prep;
1773        self.state.save()?;
1774        let repo = self.state.repo.clone();
1775        let base = self.state.base_commit.clone();
1776        let plan = refs::plan(&repo, &self.state.seeds).await?;
1777        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1778        let root = self.state.worktree_root();
1779        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1780
1781        // The hook is the write-time half of the blindness contract; the
1782        // presentation filter in `blind` is the half that cannot be bypassed.
1783        let hooks_dir = self.state.dir().join("hooks");
1784        if self.state.config.blind.commit_msg_hook {
1785            std::fs::create_dir_all(&hooks_dir)
1786                .with_context(|| format!("create {}", hooks_dir.display()))?;
1787            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1788            let path = hooks_dir.join("commit-msg");
1789            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1790            make_executable(&path)?;
1791            // Ref-counted rather than a plain idempotent set: with more than
1792            // one run able to be in flight in the same repository at once
1793            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1794            // true?" check cannot tell "another run of mine still needs
1795            // this" from "nobody does", and the run that happens to finish
1796            // first would disable the hook out from under a sibling still
1797            // relying on it.
1798            git::acquire_worktree_config(&repo).await?;
1799            self.state.enabled_worktree_config = true;
1800        }
1801
1802        // `[worktree] setup` hides its products per worktree, which needs the
1803        // same per-worktree config the hook does. Held until fold, like it.
1804        ensure_setup_config(&mut self.state, &repo).await?;
1805
1806        for (index, (spec, label)) in self
1807            .roles
1808            .implementers
1809            .clone()
1810            .into_iter()
1811            .zip(labels)
1812            .enumerate()
1813        {
1814            let branch = self.state.branch_for(label);
1815            let worktree = root.join(format!("cand-{label}"));
1816            // Persist intent before Git can leave a partially created tree.
1817            // Candidate registration follows setup, so it cannot inventory
1818            // crashes or refusals during creation itself.
1819            self.state.event(
1820                "prep-intent",
1821                serde_json::json!({
1822                    "branch": branch,
1823                    "worktree": worktree,
1824                    "start": start,
1825                    "picks": plan.picks,
1826                })
1827                .to_string(),
1828            );
1829            self.state.save()?;
1830            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1831            if self.state.config.blind.commit_msg_hook {
1832                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1833            }
1834            git::local_exclude(&worktree, "/.magi/").await?;
1835            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1836                // Nothing of this prep may survive: a resume must start over,
1837                // not find half the candidates and skip the rest.
1838                git::worktree_remove_clean(&repo, &worktree).await.ok();
1839                git::branch_delete(&repo, &branch).await.ok();
1840                self.rollback_prep(&repo).await;
1841                self.state.status = RunStatus::Blocked;
1842                self.state
1843                    .event("prep", format!("worktree setup failed: {e:#}"));
1844                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1845                self.state.save()?;
1846                return Err(e);
1847            }
1848            self.state.candidates.push(Candidate {
1849                index,
1850                label,
1851                agent: spec.id.clone(),
1852                branch,
1853                worktree,
1854                summary: String::new(),
1855                stat: String::new(),
1856                files: 0,
1857                commits: 0,
1858                empty: false,
1859                failed: None,
1860                verified_noop: None,
1861                duration_ms: 0,
1862                folded: false,
1863            });
1864            self.state.save()?;
1865            for pick in &plan.picks {
1866                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1867                {
1868                    self.state.status = RunStatus::Blocked;
1869                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1870                    self.state
1871                        .event("prep", format!("cannot apply referenced commit: {e}"));
1872                    self.state.save()?;
1873                    return Err(e);
1874                }
1875            }
1876        }
1877
1878        for j in 1..=self.roles.judges.len() {
1879            let wt = root.join(format!("judge-{j}"));
1880            if !wt.exists() {
1881                git::worktree_add_detached(&repo, &wt, &base).await?;
1882                self.setup_seat_worktree(&repo, &wt, true).await?;
1883            }
1884        }
1885
1886        // Disposable, detached checkouts for the design-deliberation stage's
1887        // advisor seats — the same shape as the judges' above, at the same
1888        // base commit, since advisors also only ever read. Sized off the
1889        // configured count directly rather than a resolved roster: unlike
1890        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1891        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1892        // `prep` has no `ResolvedRoles` field to read a count from here.
1893        if self.state.config.graph.advise {
1894            for k in 1..=self.state.config.graph.advisors {
1895                let wt = root.join(format!("advisor-{k}"));
1896                if !wt.exists() {
1897                    git::worktree_add_detached(&repo, &wt, &base).await?;
1898                    self.setup_seat_worktree(&repo, &wt, true).await?;
1899                }
1900            }
1901        }
1902
1903        // A judge cannot tell it is looking at its own patch — the seats keep
1904        // separate conversations — but a panel that shares agents with the
1905        // field is less independent than it looks, and that is worth saying out
1906        // loud once per run rather than leaving it in the config.
1907        let authors: Vec<&str> = self
1908            .roles
1909            .implementers
1910            .iter()
1911            .map(|a| a.id.as_str())
1912            .collect();
1913        let overlap: Vec<String> = self
1914            .roles
1915            .judges
1916            .iter()
1917            .enumerate()
1918            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1919            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1920            .collect();
1921        if !overlap.is_empty() {
1922            let note = format!(
1923                "{} also authored a candidate; blind, but the panel is less \
1924                 independent than {} distinct agents would be",
1925                overlap.join(", "),
1926                self.roles.judges.len()
1927            );
1928            self.state.event("prep", note);
1929        }
1930
1931        self.state.event(
1932            "prep",
1933            format!(
1934                "{} candidates, {} judges, base {} ({})",
1935                self.state.candidates.len(),
1936                self.roles.judges.len(),
1937                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1938                self.state.base_branch
1939            ),
1940        );
1941        self.state.prep_complete = Some(true);
1942        self.state.status = RunStatus::Implementing;
1943        self.state.save()?;
1944        Ok(())
1945    }
1946
1947    // -------------------------------------------------------------- advise
1948
1949    /// The design-deliberation stage: independent, read-only advisor seats
1950    /// each sketch a design before any implementer touches the repository,
1951    /// and (when at least one produced a usable proposal) a synthesis seat
1952    /// blends them into a brief `implement` carries in every candidate's
1953    /// prompt.
1954    ///
1955    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1956    /// advisors` is the proposal count. Everything here is best-effort and
1957    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1958    /// that cannot reach quota, or a synthesis seat that produced nothing
1959    /// usable all leave `implement` exactly as it was before this stage
1960    /// existed — the task instruction alone — rather than failing the whole
1961    /// competition over an enrichment stage. Every outcome is still recorded
1962    /// as an event, so a run that got nothing from this stage says why.
1963    ///
1964    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1965    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1966    /// resumed run whose stage failed would re-run it, and re-spend the
1967    /// agent calls, on every reentry before `implement`.
1968    ///
1969    /// Also skipped once any candidate shows implementation progress — the
1970    /// exact predicate `implement` itself uses to decide a candidate is no
1971    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1972    /// is not enough: a run created by an older binary that predates this
1973    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1974    /// an already-`Implementing`-or-later run under this build would
1975    /// otherwise walk straight back through `prep` (a no-op once candidates
1976    /// exist) into this node and spawn every advisor seat against worktrees
1977    /// `prep` never recreated — after implementation has already started,
1978    /// which is exactly the invariant this stage exists to guarantee.
1979    async fn advise(&mut self) -> Result<()> {
1980        let implement_untouched = self
1981            .state
1982            .candidates
1983            .iter()
1984            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1985        if !self.state.config.graph.advise || self.state.advise_attempted {
1986            return Ok(());
1987        }
1988        if !implement_untouched {
1989            self.state.event(
1990                "advise",
1991                "skipping the design-deliberation stage: at least one \
1992                 candidate already shows implementation progress, so this \
1993                 run is past the point the stage exists to run before"
1994                    .to_owned(),
1995            );
1996            self.state.advise_attempted = true;
1997            self.state.save()?;
1998            return Ok(());
1999        }
2000        let run_id = self.state.id.clone();
2001        let prompts = self.state.config.prompts.clone();
2002        let instruction = self.state.instruction.clone();
2003        let language = self.state.config.graph.language.clone();
2004        let root = self.state.worktree_root();
2005        let n = self.state.config.graph.advisors;
2006        let where_recorded = self.state.dir().join("run.json");
2007
2008        let seats = match self.state.config.advisors() {
2009            Ok(seats) if !seats.is_empty() => seats,
2010            Ok(_) => {
2011                self.state.event(
2012                    "advise",
2013                    format!(
2014                        "[graph] advisors is 0; skipping the design-deliberation \
2015                         stage and continuing without a synthesis brief (see {})",
2016                        where_recorded.display()
2017                    ),
2018                );
2019                self.state.advise_attempted = true;
2020                self.state.save()?;
2021                return Ok(());
2022            }
2023            Err(e) => {
2024                self.state.event(
2025                    "advise",
2026                    format!(
2027                        "could not resolve advisor seats ({e:#}); continuing \
2028                         without a design-deliberation brief (see {})",
2029                        where_recorded.display()
2030                    ),
2031                );
2032                self.state.advise_attempted = true;
2033                self.state.save()?;
2034                return Ok(());
2035            }
2036        };
2037
2038        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2039        let artifacts = agent::artifacts_dir(&self.state.dir());
2040        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2041
2042        let mut jobs = Vec::new();
2043        for (i, spec) in seats.iter().cloned().enumerate() {
2044            let seat_key = format!("advisor-{}", i + 1);
2045            let seat = self.seat(&seat_key, &spec.id);
2046            jobs.push(SeatJob {
2047                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2048                spec,
2049                seat,
2050                cwd: worktrees[i % worktrees.len()].clone(),
2051                timeout,
2052                allow_write: false,
2053                sessions: false,
2054                artifacts: artifacts.clone(),
2055                stem: seat_key,
2056                handover: None,
2057            });
2058        }
2059
2060        self.state.event(
2061            "advise",
2062            format!(
2063                "{} advisor seat(s) sketching a design in parallel",
2064                jobs.len()
2065            ),
2066        );
2067        let mut quota_losses = Vec::new();
2068        let cache = self.state.config.cache_dir();
2069        let ctx = WaveCtx {
2070            carry_seats: false,
2071            pauses: Some((&self.pause, &self.interrupt)),
2072            run: &run_id,
2073            node: "advise",
2074            prompts: &prompts,
2075            cache: cache.as_deref(),
2076            round: None,
2077        };
2078        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2079        let results = ask_json_wave::<Proposal>(
2080            jobs,
2081            Arc::clone(&self.sem),
2082            self.state.config.graph.retries,
2083            &advisor_roster,
2084            &ctx,
2085            &mut quota_losses,
2086            &mut self.state,
2087            &|p: &Proposal| p.validate(),
2088        )
2089        .await;
2090        if self.state.resource_block.is_some() {
2091            return self.stop_for_resource();
2092        }
2093        self.state.quota.extend(quota_losses);
2094
2095        let mut records = Vec::with_capacity(results.len());
2096        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2097            let agent_id = seat.agent.clone();
2098            self.state.seats.insert(seat.key.clone(), seat);
2099            match res {
2100                Ok((proposal, out)) => {
2101                    self.state
2102                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2103                    records.push(advise::AdvisorRecord::proposed(
2104                        i + 1,
2105                        agent_id,
2106                        proposal,
2107                        out.duration_ms,
2108                    ));
2109                }
2110                Err(e) => {
2111                    self.state.event(
2112                        "advise",
2113                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2114                    );
2115                    records.push(advise::AdvisorRecord::failed(
2116                        i + 1,
2117                        agent_id,
2118                        e.to_string(),
2119                    ));
2120                }
2121            }
2122        }
2123
2124        let mut advice = advise::Advice {
2125            records,
2126            synthesis: None,
2127        };
2128        if advice.proposals().is_empty() {
2129            self.state.event(
2130                "advise",
2131                "no advisor produced a usable proposal; continuing without a \
2132                 synthesis brief"
2133                    .to_owned(),
2134            );
2135        } else {
2136            match self
2137                .synthesize_brief(
2138                    &advice,
2139                    &instruction,
2140                    &language,
2141                    &worktrees[0],
2142                    &artifacts,
2143                    &run_id,
2144                    &prompts,
2145                    cache.as_deref(),
2146                )
2147                .await
2148            {
2149                Ok(Some(text)) => {
2150                    self.state.event(
2151                        "advise",
2152                        "synthesized a design brief for the implementer".to_owned(),
2153                    );
2154                    advice.synthesis = Some(text);
2155                }
2156                Ok(None) => {
2157                    self.state.event(
2158                        "advise",
2159                        "the synthesis seat produced nothing usable; continuing \
2160                         without a design brief"
2161                            .to_owned(),
2162                    );
2163                }
2164                Err(e) => {
2165                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2166                        return Err(e);
2167                    }
2168                    self.state.event(
2169                        "advise",
2170                        format!("could not synthesize a design brief: {e:#}"),
2171                    );
2172                }
2173            }
2174        }
2175        advise::apply_reflection(&mut advice);
2176
2177        self.state.advice = Some(advice);
2178        self.state.advise_attempted = true;
2179        self.state.save()?;
2180        Ok(())
2181    }
2182
2183    /// The synthesis seat: reads every advisor's proposal and blends them
2184    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2185    /// out of [`Runner::advise`] only for readability — it is not called
2186    /// anywhere else.
2187    ///
2188    /// Picked the same way [`crate::talk`]'s standing conversation and
2189    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2190    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2191    /// order (a claude seat, else the first runnable agent in roster order)
2192    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2193    /// in [`crate::config`] for why a dedicated field exists here at all.
2194    #[allow(clippy::too_many_arguments)]
2195    async fn synthesize_brief(
2196        &mut self,
2197        advice: &advise::Advice,
2198        instruction: &str,
2199        language: &str,
2200        cwd: &Path,
2201        artifacts: &Path,
2202        run_id: &str,
2203        prompts: &Prompts,
2204        cache: Option<&Path>,
2205    ) -> Result<Option<String>> {
2206        let chain = agent::pick_chain(
2207            &self.state.config.agents,
2208            self.state.config.roles.synthesizer.as_ref(),
2209            &agent::installed,
2210            "synthesizer",
2211        )?;
2212        let proposals = advice.proposals();
2213        let mut prompt = prompt::with_overlay(
2214            prompt::synthesize_brief(instruction, &proposals, language),
2215            prompts.overlay("advise"),
2216        );
2217        if cache.is_some() {
2218            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2219            // below — see `prompt::build_cache_note`'s doc for why telling a
2220            // read-only seat to build through the shared cache is exactly how
2221            // a sandbox's write refusal gets misread as a defect.
2222            prompt.push('\n');
2223            prompt.push_str(&prompt::build_cache_note("advise", false));
2224        }
2225        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2226        // Each id is tried once, in order; a quota hit, error or unusable
2227        // answer moves to the next. The seat is single-turn (`sessions:
2228        // false`) and the prompt is the whole context, so a fallback agent
2229        // needs nothing carried over.
2230        let mut last = None;
2231        for (n, spec) in chain.iter().enumerate() {
2232            if n > 0 {
2233                self.state
2234                    .event("advise", format!("synthesis falling back to {}", spec.id));
2235            }
2236            let mut seat = self.seat("advise-synthesis", &spec.id);
2237            let wait = self.repository_wait();
2238            let outcome = wait
2239                .scope(agent::invoke(
2240                    spec,
2241                    &mut seat,
2242                    &Invocation {
2243                        scope: crate::agent::InvocationScope::Repository,
2244                        coordination: self.state.config.coordination.as_ref(),
2245                        fetched_base: None,
2246                        cwd,
2247                        prompt: &prompt,
2248                        timeout,
2249                        allow_write: false,
2250                        unsandboxed: false,
2251                        sessions: false,
2252                        artifacts,
2253                        stem: &if n == 0 {
2254                            "advise-synthesis".to_owned()
2255                        } else {
2256                            format!("advise-synthesis-{}", spec.id)
2257                        },
2258                        run: run_id,
2259                        node: "advise",
2260                        cache_dir: None,
2261                        attachments: &[],
2262                        writable: &[],
2263                    },
2264                ))
2265                .await;
2266            if outcome.is_ok() {
2267                self.state.seats.insert(seat.key.clone(), seat);
2268            }
2269            let advance = agent::chain_advances(&outcome);
2270            last = Some(outcome);
2271            if !advance {
2272                break;
2273            }
2274        }
2275        // Exhausted: the last attempt's result is what a single failed seat
2276        // would have produced.
2277        let out = last.expect("a chain holds at least one agent")?;
2278        if !out.usable() {
2279            return Ok(None);
2280        }
2281        let text =
2282            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2283        Ok((!text.trim().is_empty()).then_some(text))
2284    }
2285
2286    // ----------------------------------------------------------- implement
2287
2288    async fn implement(&mut self) -> Result<()> {
2289        self.persist_candidates().await?;
2290        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2291        // agent files with `magi task add` name the run that paid for it. The
2292        // prompt overlay is cloned alongside it because the waves borrow it
2293        // while `self` is mutably borrowed by the node's own bookkeeping.
2294        let run_id = self.state.id.clone();
2295        let prompts = self.state.config.prompts.clone();
2296        let todo: Vec<usize> = self
2297            .state
2298            .candidates
2299            .iter()
2300            .enumerate()
2301            .filter(|(_, c)| {
2302                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2303                    && c.failed.is_none()
2304                    && !c.empty
2305            })
2306            .map(|(i, _)| i)
2307            .collect();
2308        if todo.is_empty() {
2309            return self.after_implement();
2310        }
2311        self.state.status = RunStatus::Implementing;
2312
2313        let language = self.state.config.graph.language.clone();
2314        let github_language = self.state.config.graph.github_language.clone();
2315        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2316        let sessions = self.state.config.graph.sessions;
2317        let artifacts = agent::artifacts_dir(&self.state.dir());
2318        // The design-deliberation stage's blended brief, when `advise` found
2319        // one — carried into every implementer's prompt the same way
2320        // regardless of which candidate it is.
2321        let brief = self
2322            .state
2323            .advice
2324            .as_ref()
2325            .and_then(|a| a.synthesis.as_deref())
2326            .map(str::to_owned);
2327        let attachments = self.state.attachments.clone();
2328
2329        let mut jobs = Vec::new();
2330        for &i in &todo {
2331            let (index, label, worktree) = {
2332                let c = &self.state.candidates[i];
2333                (c.index, c.label, c.worktree.clone())
2334            };
2335            let spec = self.roles.implementers[index].clone();
2336            let seat_key = format!("impl-{label}");
2337            let seat = self.seat(&seat_key, &spec.id);
2338            let instruction = seeded_instruction(&self.state);
2339            jobs.push(SeatJob {
2340                spec,
2341                seat,
2342                prompt: prompt::implement_in(
2343                    &instruction,
2344                    &worktree.to_string_lossy(),
2345                    &language,
2346                    &github_language,
2347                    brief.as_deref(),
2348                    &attachments,
2349                ),
2350                cwd: worktree,
2351                timeout,
2352                allow_write: true,
2353                sessions,
2354                artifacts: artifacts.clone(),
2355                stem: format!("impl-{label}"),
2356                handover: None,
2357            });
2358        }
2359
2360        self.state.event(
2361            "implement",
2362            format!("{} candidates in parallel", jobs.len()),
2363        );
2364        // Kept so a seat whose CLI hung up can be asked again from the same
2365        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2366        // can update a seat's own entry once a fallback agent takes it over —
2367        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2368        // whichever agent actually answered, not the one that quota'd out.
2369        let mut sent = jobs.clone();
2370        let cache = self.state.config.cache_dir();
2371        let ctx = WaveCtx {
2372            carry_seats: false,
2373            pauses: Some((&self.pause, &self.interrupt)),
2374            run: &run_id,
2375            node: "implement",
2376            prompts: &prompts,
2377            cache: cache.as_deref(),
2378            round: None,
2379        };
2380        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2381        if !wave_blocked(&results) {
2382            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2383                .await;
2384            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2385                .await?;
2386            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2387                .await;
2388        }
2389        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2390
2391        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2392            && results
2393                .iter()
2394                .any(|(_, _, out)| FailClass::of(out).is_some());
2395        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2396            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2397                // Executed turns interrupted by disk pressure already carry
2398                // a preservation barrier from wave. Admission-only waits do
2399                // not. Keep the interrupted seat's session for recovery too.
2400                if self
2401                    .state
2402                    .candidate_preservation
2403                    .contains_key(&self.state.candidates[i].label)
2404                {
2405                    self.state.seats.insert(seat.key.clone(), seat);
2406                }
2407                continue;
2408            }
2409            let seat_key = seat.key.clone();
2410            // A quota fallback (`resume_seat_handovers`) may have handed this
2411            // seat to a different agent than the one `prep` recorded on the
2412            // candidate; the stats tables and any later fixer-defaults-to-
2413            // winner's-author lookup must credit whoever actually answered —
2414            // unless every fallback also quota'd out, in which case nobody
2415            // actually answered and crediting the last agent tried would
2416            // erase every earlier agent's own quota loss from the stats
2417            // tables instead of just this one seat's.
2418            let agent = seat.agent.clone();
2419            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2420            self.state.seats.insert(seat.key.clone(), seat);
2421            let label = self.state.candidates[i].label;
2422            self.state.interrupted_implementations.remove(&label);
2423
2424            let (summary, duration, failed, verified_claim) = match out {
2425                AgentOutcome::Ok(o) => {
2426                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2427                    let failed = (!o.usable()).then(|| {
2428                        if o.timed_out {
2429                            "agent timed out".to_owned()
2430                        } else {
2431                            format!(
2432                                "agent exited with {}",
2433                                crate::notices::exit_status(o.exit_code)
2434                            )
2435                        }
2436                    });
2437                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2438                    (text, o.duration_ms, failed, verified_claim)
2439                }
2440                // Left un-resumed by `resume_undelivered` (a dirty tree
2441                // already rescues the work, or there was no session left to
2442                // resume into) — reported like the ordinary failure it is,
2443                // never as if `o.text` (the CLI's raw error JSON) were an
2444                // answer.
2445                AgentOutcome::Dropped(o) => {
2446                    let why = o
2447                        .dropped
2448                        .as_ref()
2449                        .map(|d| d.why.as_str())
2450                        .unwrap_or("the CLI ended the stream without delivering its answer");
2451                    (
2452                        String::new(),
2453                        o.duration_ms,
2454                        Some(format!("the CLI dropped the stream ({why})")),
2455                        None,
2456                    )
2457                }
2458                AgentOutcome::Quota(o) => {
2459                    self.state.quota.push(QuotaLoss {
2460                        seat: seat_key,
2461                        node: "implement".to_owned(),
2462                        at: Timestamp::now(),
2463                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2464                    });
2465                    (
2466                        String::new(),
2467                        o.duration_ms,
2468                        Some("rate limited (quota); produced no change".to_owned()),
2469                        None,
2470                    )
2471                }
2472                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2473                    (String::new(), 0, Some(e), None)
2474                }
2475            };
2476
2477            // Save every completed turn before any fallible Git operation. A failed
2478            // rescue must not erase the reply or prevent collection of other seats.
2479            let c = &mut self.state.candidates[i];
2480            if !exhausted_the_fallback_chain {
2481                c.agent = agent;
2482            }
2483            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2484            c.duration_ms = duration;
2485            c.failed = failed;
2486            c.verified_noop = verified_claim;
2487            self.state
2488                .candidate_preservation
2489                .insert(label, "awaiting persistence".to_owned());
2490        }
2491        self.state.save()?;
2492        self.persist_candidates().await?;
2493
2494        if resource_blocked {
2495            return self.stop_for_resource();
2496        }
2497        if selected_failed {
2498            self.state.status = RunStatus::Blocked;
2499            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2500            self.state.save()?;
2501            bail!(
2502                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2503            );
2504        }
2505        self.after_implement()
2506    }
2507
2508    /// Retry only persistence after a completed implementation turn.
2509    async fn persist_candidates(&mut self) -> Result<()> {
2510        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2511        for label in labels {
2512            let i = self
2513                .state
2514                .candidates
2515                .iter()
2516                .position(|c| c.label == label)
2517                .context("pending candidate is missing")?;
2518            let c = self.state.candidates[i].clone();
2519            let interrupted = self.state.interrupted_implementations.contains(&label);
2520            let result = async {
2521                let wait = crate::repo_lock::WaitPolicy::new(
2522                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2523                    || false,
2524                );
2525                let _coordination =
2526                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2527                // Best effort evidence requires no index writes. A successful snapshot
2528                // never authorizes removal of a tree whose rescue failed.
2529                if let Err(e) = git::snapshot_uncommitted(
2530                    &c.worktree,
2531                    &self
2532                        .state
2533                        .dir()
2534                        .join("artifacts")
2535                        .join(format!("candidate-{label}-recovery")),
2536                )
2537                .await
2538                {
2539                    self.state.event(
2540                        "preserve",
2541                        format!("candidate {label} snapshot incomplete: {e:#}"),
2542                    );
2543                }
2544                let rescued = git::rescue_commit(
2545                    &c.worktree,
2546                    &format!("magi: candidate {label} (uncommitted work)"),
2547                )
2548                .await?;
2549                self.state.note_withheld("implement", &rescued.withheld);
2550                let base = &self.state.base_commit;
2551                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2552                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2553                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2554                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2555                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2556                if c.empty && commits > 0 && !patch.trim().is_empty()
2557                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2558                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2559                        || self.state.tally.is_some())
2560                {
2561                    // Keep the old classification and pending marker so every
2562                    // retry stops too. Retain decisions as recovery evidence;
2563                    // only a supported recovery operation may replace them.
2564                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2565                }
2566                let c = &mut self.state.candidates[i];
2567                c.commits = commits;
2568                c.stat = stat;
2569                c.files = files;
2570                let no_changes = commits == 0 || patch.trim().is_empty();
2571                c.empty = no_changes && !interrupted;
2572                if interrupted && no_changes {
2573                    c.failed = None;
2574                    c.verified_noop = None;
2575                }
2576                if !c.empty {
2577                    c.failed = None;
2578                    c.verified_noop = None;
2579                }
2580                self.state.event(
2581                    "preserve",
2582                    format!("candidate {label}: {files} files, {commits} commits"),
2583                );
2584                Ok::<_, anyhow::Error>(())
2585            }
2586            .await;
2587            match result {
2588                Ok(()) => {
2589                    self.state.candidate_preservation.remove(&label);
2590                    if !interrupted || self.state.candidates[i].files > 0 {
2591                        self.state.interrupted_implementations.remove(&label);
2592                    }
2593                }
2594                Err(e) => {
2595                    let reason = format!("candidate {label} preservation failed: {e:#}");
2596                    self.state.event("preserve", &reason);
2597                    self.state.candidate_preservation.insert(label, reason);
2598                }
2599            }
2600            if self.state.candidate_preservation.is_empty()
2601                && self.state.status == RunStatus::Blocked
2602                && self.state.candidates.iter().all(|c| c.failed.is_none())
2603            {
2604                // Persist the recovered phase with the cleared pending marker;
2605                // a crash between saves must not turn this into a selected-seat
2606                // failure hold on the next resume.
2607                self.state.status = RunStatus::Implementing;
2608                self.state.event(
2609                    "preserve",
2610                    "candidate persistence recovered; implementation can continue",
2611                );
2612            }
2613            self.state.save()?;
2614        }
2615        if !self.state.candidate_preservation.is_empty() {
2616            self.state.status = RunStatus::Blocked;
2617            self.state.save()?;
2618            bail!(
2619                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2620            );
2621        }
2622        Ok(())
2623    }
2624
2625    /// Ask again, once, for work a CLI did and then failed to hand over.
2626    ///
2627    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2628    /// status with an empty response and a usage report showing output tokens,
2629    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2630    /// seven minutes and 14,267 output tokens that arrived as an empty
2631    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2632    ///
2633    /// Two conditions, and both matter:
2634    ///
2635    /// - **Only when the tree is untouched.** Often the agent has already
2636    ///   written its files and only the closing message was lost; the rescue
2637    ///   commit below picks that up and there is nothing to ask for. Re-asking
2638    ///   then would pay for a second implementation of work already on disk.
2639    /// - **Once.** A CLI that drops one stream can drop the next, and this
2640    ///   node is the most expensive in the graph.
2641    ///
2642    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2643    /// dropped reply still carried its `conversation_id`, so the seat is asked
2644    /// to finish what it was doing rather than sent the whole task again. It
2645    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2646    /// node's - for the same reason a re-ranked judge does: restating finished
2647    /// work is not the work.
2648    ///
2649    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2650    /// same way until it resets, while an abandoned conversation is still
2651    /// there to be picked up.
2652    async fn resume_undelivered(
2653        &mut self,
2654        results: &mut [(usize, SeatState, AgentOutcome)],
2655        sent: &[SeatJob],
2656        prompts: &Prompts,
2657        run_id: &str,
2658    ) {
2659        for (wi, seat, out) in results.iter_mut() {
2660            let Some(dropped) = (match &*out {
2661                AgentOutcome::Dropped(o) => o.dropped.clone(),
2662                _ => None,
2663            }) else {
2664                continue;
2665            };
2666            let Some(job) = sent.get(*wi) else { continue };
2667            // Already on disk? Then only the closing message was lost.
2668            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2669                self.state.event(
2670                    "implement",
2671                    format!(
2672                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2673                         work is in the tree",
2674                        seat.key, dropped.output_tokens, dropped.why
2675                    ),
2676                );
2677                continue;
2678            }
2679            // The re-ask only makes sense as a resume: `resume_after_drop`
2680            // says nothing about the task, trusting the seat to still hold it.
2681            // Without a session to resume — sessions disabled, or this CLI's
2682            // drop shape happened not to carry a session id — that prompt
2683            // would open a brand-new conversation with no context at all,
2684            // which is worse than leaving this as the ordinary failure it
2685            // already is.
2686            if !has_context(&job.spec, seat, job.sessions) {
2687                self.state.event(
2688                    "implement",
2689                    format!(
2690                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2691                         is no session left to resume",
2692                        seat.key, dropped.output_tokens, dropped.why
2693                    ),
2694                );
2695                continue;
2696            }
2697            self.state.event(
2698                "implement",
2699                format!(
2700                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2701                     conversation",
2702                    seat.key, dropped.output_tokens, dropped.why
2703                ),
2704            );
2705            let mut retry = job.clone();
2706            retry.seat = seat.clone();
2707            retry.prompt = prompt::resume_after_drop(&dropped.why);
2708            retry.timeout = retry_budget(job.timeout, true);
2709            retry.stem = format!("{}-resume", job.stem);
2710            let cache = self.state.config.cache_dir();
2711            let ctx = WaveCtx {
2712                carry_seats: false,
2713                pauses: Some((&self.pause, &self.interrupt)),
2714                run: run_id,
2715                node: "implement",
2716                prompts,
2717                cache: cache.as_deref(),
2718                round: None,
2719            };
2720            let (resumed_seat, resumed) =
2721                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2722            *seat = resumed_seat;
2723            *out = resumed;
2724        }
2725    }
2726
2727    /// Fall an implement seat through to the next untried agent in the
2728    /// implementer roster when it lost to quota — or, since the handover was
2729    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2730    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2731    /// quota path itself being unchanged — instead of leaving the
2732    /// seat's loss final the moment one agent's account runs dry.
2733    ///
2734    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2735    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2736    /// truncates to the single slot rotation picked, so a solo task whose one
2737    /// implementer hits quota mid-run used to have nothing else to try. This
2738    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2739    /// unrotated roster — which is the only place the *other* candidates in
2740    /// the machine's roster still exist once `implementers` has been cut down
2741    /// to size.
2742    ///
2743    /// Walks forward from just past the seat's own original position in the
2744    /// roster, never wrapping back to the front: a later candidate slot (say
2745    /// `beta`, the roster's second entry) must fall through to the *next*
2746    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2747    /// almost certainly a different candidate's own agent already — and once
2748    /// the roster's tail is exhausted there is nothing left to fall through
2749    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2750    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2751    /// let this retry that id forever. The loop keeps falling through until
2752    /// an attempt lands something other than `Quota` or the roster's tail
2753    /// runs out of untried ids, at which point the seat is left exactly as
2754    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2755    /// `QuotaLoss` recorded, the candidate failed/empty.
2756    ///
2757    /// `sent` is taken mutably and updated with the fallback agent's spec:
2758    /// `resume_unconfirmed_commands`, which runs after this and also reads
2759    /// `sent`, must see whichever agent actually ended up answering the seat
2760    /// — reading the stale, original spec there would check session
2761    /// eligibility against the wrong CLI and could hand a fallback agent's
2762    /// session id to the agent that just lost the seat to quota.
2763    ///
2764    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2765    /// — `self.seat` only reuses state when the agent id is unchanged, so
2766    /// handing it a different id already gets this for free. Reusing the old
2767    /// seat would resume a different CLI's session as if it were a
2768    /// continuation of this one.
2769    ///
2770    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2771    /// a quota loss cuts an agent off mid-turn, so anything already in the
2772    /// tree is unfinished work, not a completed candidate a re-ask would pay
2773    /// for twice. A dirty tree is rescued into a commit first (the same
2774    /// neutral-identity rescue `implement`'s own outcome loop gives every
2775    /// candidate) so the next agent starts clean.
2776    ///
2777    /// The new agent gets the implementer's full prompt and full
2778    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2779    /// it has no session and no context, and is implementing the task from
2780    /// nothing, unlike a resumed drop which is only restating work already
2781    /// done.
2782    ///
2783    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2784    /// `implement` event, never into `self.state.quota` — that is what
2785    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2786    /// go unspent, and a seat that ultimately recovered on its second or
2787    /// third agent is not the stalled panel that check exists to catch. Only
2788    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2789    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2790    /// outcome loop already has — this helper never pushes to it itself.
2791    async fn resume_seat_handovers(
2792        &mut self,
2793        results: &mut [(usize, SeatState, AgentOutcome)],
2794        sent: &mut [SeatJob],
2795        prompts: &Prompts,
2796        run_id: &str,
2797    ) -> Result<()> {
2798        let instruction = seeded_instruction(&self.state);
2799        let language = self.state.config.graph.language.clone();
2800        let github_language = self.state.config.graph.github_language.clone();
2801        let brief = self
2802            .state
2803            .advice
2804            .as_ref()
2805            .and_then(|a| a.synthesis.as_deref())
2806            .map(str::to_owned);
2807        let attachments = self.state.attachments.clone();
2808        for (wi, seat, out) in results.iter_mut() {
2809            // Who holds the other candidate seats of this wave right now
2810            // (earlier handovers already written back to `sent`).
2811            let others: BTreeSet<String> = sent
2812                .iter()
2813                .enumerate()
2814                .filter(|(j, _)| j != wi)
2815                .map(|(_, j)| j.spec.id.clone())
2816                .collect();
2817            let Some(job) = sent.get_mut(*wi) else {
2818                continue;
2819            };
2820            // Where the seat's own original agent sits in the roster — the
2821            // fallback walk starts just past here, never at the front, so a
2822            // later candidate slot's quota loss does not fall back onto an
2823            // earlier slot's own agent.
2824            let start = self
2825                .roles
2826                .implementer_roster
2827                .iter()
2828                .position(|s| s.id == job.spec.id)
2829                .unwrap_or(0);
2830            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2831            let mut fallback_attempt = 0usize;
2832            let mut prev: Option<FailClass> = None;
2833            while let Some(cur) = FailClass::of(&*out) {
2834                if !should_hand_over(prev.as_ref(), &cur) {
2835                    break;
2836                }
2837                let Some(next) =
2838                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2839                        .cloned()
2840                else {
2841                    break;
2842                };
2843                tried.insert(next.id.clone());
2844                fallback_attempt += 1;
2845
2846                let coordination = self.coordinate(&job.cwd).await?;
2847                if let Ok(r) = git::rescue_commit(
2848                    &job.cwd,
2849                    &format!(
2850                        "magi: candidate {} (uncommitted work before {} fallback)",
2851                        seat.key,
2852                        if cur == FailClass::Quota {
2853                            "quota"
2854                        } else {
2855                            "handover"
2856                        }
2857                    ),
2858                )
2859                .await
2860                {
2861                    self.state.note_withheld("implement", &r.withheld);
2862                }
2863                drop(coordination);
2864
2865                record_handover(
2866                    &mut self.state,
2867                    "implement",
2868                    &seat.key,
2869                    &seat.agent,
2870                    &next.id,
2871                    &cur,
2872                    &fail_reason(&*out),
2873                );
2874                prev = Some(cur.clone());
2875
2876                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2877                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2878                // Kept in sync on `sent` itself, not just the local retry: a
2879                // later helper (`resume_unconfirmed_commands`) reads `sent`
2880                // after this one returns and must see whichever agent is now
2881                // occupying the seat, not the one that just quota'd out —
2882                // otherwise it would judge session/continuation eligibility
2883                // by the wrong CLI and could resend a fallback's session id
2884                // to the agent that lost it the seat in the first place.
2885                job.spec = next.clone();
2886                let mut retry = job.clone();
2887                retry.seat = new_seat;
2888                retry.prompt = prompt::implement_in(
2889                    &instruction,
2890                    &job.cwd.to_string_lossy(),
2891                    &language,
2892                    &github_language,
2893                    brief.as_deref(),
2894                    &attachments,
2895                );
2896                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2897                let cache = self.state.config.cache_dir();
2898                let ctx = WaveCtx {
2899                    carry_seats: false,
2900                    pauses: Some((&self.pause, &self.interrupt)),
2901                    run: run_id,
2902                    node: "implement",
2903                    prompts,
2904                    cache: cache.as_deref(),
2905                    round: None,
2906                };
2907                let (fallback_seat, fallback_out) = run_one(
2908                    retry,
2909                    Arc::clone(&self.sem),
2910                    &ctx,
2911                    &mut self.state,
2912                    fallback_attempt,
2913                )
2914                .await;
2915                *seat = fallback_seat;
2916                *out = fallback_out;
2917            }
2918        }
2919        Ok(())
2920    }
2921
2922    /// Ask an implement seat's own CLI to confirm what it started, once, when
2923    /// its reply reported a command whose completion status it never
2924    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2925    /// that does and does not mean.
2926    ///
2927    /// The completion contract this task asks for, extended to `implement`
2928    /// with the same signal `continue_fix_report` reads for the fixer,
2929    /// rather than a keyword search over the reply or a hard requirement on
2930    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2931    /// a candidate's CLI turn ended cleanly while a test run it had started
2932    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2933    /// named in it at all is untouched by this: `commands` is empty, so
2934    /// there is nothing to be unconfirmed.
2935    ///
2936    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2937    /// this is not about recovering edits that might already be on disk, it
2938    /// is about a result the seat itself never vouched for, which resuming
2939    /// asks for regardless of what the tree already holds. Bounded to one
2940    /// attempt for the same reason `resume_undelivered` is — this is the
2941    /// most expensive node in the graph — and a seat that still cannot
2942    /// confirm on that attempt is left as whatever its (possibly still
2943    /// unconfirmed) reply says; this does not invent a new "failed" reason
2944    /// for a candidate that otherwise produced a real, committed change.
2945    async fn resume_unconfirmed_commands(
2946        &mut self,
2947        results: &mut [(usize, SeatState, AgentOutcome)],
2948        sent: &[SeatJob],
2949        prompts: &Prompts,
2950        run_id: &str,
2951    ) {
2952        for (wi, seat, out) in results.iter_mut() {
2953            let AgentOutcome::Ok(o) = &*out else {
2954                continue;
2955            };
2956            if !has_unconfirmed_command(&o.commands) {
2957                continue;
2958            }
2959            let Some(job) = sent.get(*wi) else { continue };
2960            if !has_context(&job.spec, seat, job.sessions) {
2961                self.state.event(
2962                    "implement",
2963                    format!(
2964                        "{}: the reply named a command whose own CLI never confirmed the exit \
2965                         status of, but there is no session left to resume",
2966                        seat.key
2967                    ),
2968                );
2969                continue;
2970            }
2971            self.state.event(
2972                "implement",
2973                format!(
2974                    "{}: the reply named a command whose own CLI never confirmed the exit \
2975                     status of; resuming the conversation",
2976                    seat.key
2977                ),
2978            );
2979            let mut retry = job.clone();
2980            retry.seat = seat.clone();
2981            retry.prompt = prompt::resume_incomplete(
2982                "a command in your last reply had no confirmed exit status",
2983            );
2984            retry.timeout = retry_budget(job.timeout, true);
2985            retry.stem = format!("{}-confirm", job.stem);
2986            let cache = self.state.config.cache_dir();
2987            let ctx = WaveCtx {
2988                carry_seats: false,
2989                pauses: Some((&self.pause, &self.interrupt)),
2990                run: run_id,
2991                node: "implement",
2992                prompts,
2993                cache: cache.as_deref(),
2994                round: None,
2995            };
2996            let (resumed_seat, resumed) =
2997                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2998            *seat = resumed_seat;
2999            *out = resumed;
3000        }
3001    }
3002
3003    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3004    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3005    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3006    /// that motivated this.
3007    ///
3008    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3009    /// own nudge loop already covers for judge/review/vote seats, and not a
3010    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3011    /// implement seats: here the CLI turn genuinely finished while the node's
3012    /// own work — the fixer's account of what it did — had not. Gated purely
3013    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3014    /// reply, never on any wording in it, so a fixer whose valid, first-try
3015    /// `FixReport` happens to mention having waited on a background test is
3016    /// never resumed — the `Ok(report)` branch at the call site returns
3017    /// before this is ever invoked.
3018    ///
3019    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3020    /// attempt ([`retry_budget`]), nothing attempted once the session is
3021    /// gone, and a quota hit ends the loop immediately rather than retrying a
3022    /// rate limit that fails the same way again.
3023    async fn continue_fix_report(
3024        &mut self,
3025        mut seat: SeatState,
3026        parse_err: String,
3027        job: &SeatJob,
3028        prompts: &Prompts,
3029        run_id: &str,
3030        round: usize,
3031    ) -> (
3032        SeatState,
3033        Option<FixReport>,
3034        Option<String>,
3035        ContinuationRecord,
3036    ) {
3037        let mut last_err = parse_err;
3038        let mut cumulative_wait_ms = 0u64;
3039        let mut attempts = 0usize;
3040        loop {
3041            if !has_context(&job.spec, &seat, job.sessions) {
3042                self.state.event(
3043                    "fix",
3044                    format!(
3045                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3046                         session left to resume into"
3047                    ),
3048                );
3049                let outcome = if attempts == 0 {
3050                    ContinuationOutcome::NoSession
3051                } else {
3052                    ContinuationOutcome::Exhausted
3053                };
3054                return (
3055                    seat,
3056                    None,
3057                    Some(format!("unparsable fix report: {last_err}")),
3058                    ContinuationRecord {
3059                        attempts,
3060                        cumulative_wait_ms,
3061                        outcome,
3062                    },
3063                );
3064            }
3065            if attempts >= MAX_FIX_CONTINUATIONS {
3066                self.state.event(
3067                    "fix",
3068                    format!(
3069                        "round {round}: fixer's reply still had no adoption report after \
3070                         {attempts} continuation(s) ({last_err}); giving up"
3071                    ),
3072                );
3073                return (
3074                    seat,
3075                    None,
3076                    Some(format!(
3077                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3078                    )),
3079                    ContinuationRecord {
3080                        attempts,
3081                        cumulative_wait_ms,
3082                        outcome: ContinuationOutcome::Exhausted,
3083                    },
3084                );
3085            }
3086            attempts += 1;
3087            self.state.event(
3088                "fix",
3089                format!(
3090                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3091                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3092                ),
3093            );
3094            let mut retry = job.clone();
3095            retry.seat = seat.clone();
3096            retry.prompt = prompt::resume_incomplete(&last_err);
3097            retry.timeout = retry_budget(job.timeout, true);
3098            retry.stem = format!("{}-continue{attempts}", job.stem);
3099            let cache = self.state.config.cache_dir();
3100            let ctx = WaveCtx {
3101                carry_seats: false,
3102                pauses: Some((&self.pause, &self.interrupt)),
3103                run: run_id,
3104                node: "fix",
3105                prompts,
3106                cache: cache.as_deref(),
3107                round: Some(round),
3108            };
3109            let (resumed_seat, resumed_out) = run_one(
3110                retry,
3111                Arc::clone(&self.sem),
3112                &ctx,
3113                &mut self.state,
3114                attempts,
3115            )
3116            .await;
3117            seat = resumed_seat;
3118            match resumed_out {
3119                AgentOutcome::Ok(o) => {
3120                    cumulative_wait_ms += o.duration_ms;
3121                    match verdict::extract_json::<FixReport>(&o.text) {
3122                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3123                            self.state.event(
3124                                "fix",
3125                                format!(
3126                                    "round {round}: fixer's adoption report recovered after \
3127                                     {attempts} continuation(s)"
3128                                ),
3129                            );
3130                            return (
3131                                seat,
3132                                Some(report),
3133                                None,
3134                                ContinuationRecord {
3135                                    attempts,
3136                                    cumulative_wait_ms,
3137                                    outcome: ContinuationOutcome::Resumed,
3138                                },
3139                            );
3140                        }
3141                        // The report parsed, but this same reply's own
3142                        // CommandEvidence — the identical record `state.jobs`
3143                        // renders — names a command whose CLI never
3144                        // confirmed an exit status. Read together, that is
3145                        // not a resolved answer: keep nudging rather than
3146                        // accept a report standing next to a command the
3147                        // seat's own CLI cannot vouch for.
3148                        Ok(_) => {
3149                            last_err = "the reply parsed, but it reported a command whose own CLI \
3150                                 never confirmed an exit status"
3151                                .to_owned();
3152                        }
3153                        Err(e) => last_err = e.to_string(),
3154                    }
3155                }
3156                AgentOutcome::Quota(o) => {
3157                    cumulative_wait_ms += o.duration_ms;
3158                    self.state.quota.push(QuotaLoss {
3159                        seat: seat.key.clone(),
3160                        node: "fix".to_owned(),
3161                        at: Timestamp::now(),
3162                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3163                    });
3164                    self.state.event(
3165                        "fix",
3166                        format!(
3167                            "round {round}: continuation rate limited (quota); not retrying now"
3168                        ),
3169                    );
3170                    return (
3171                        seat,
3172                        None,
3173                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3174                        ContinuationRecord {
3175                            attempts,
3176                            cumulative_wait_ms,
3177                            outcome: ContinuationOutcome::QuotaLost,
3178                        },
3179                    );
3180                }
3181                AgentOutcome::Dropped(o) => {
3182                    cumulative_wait_ms += o.duration_ms;
3183                    let why = o
3184                        .dropped
3185                        .as_ref()
3186                        .map(|d| d.why.as_str())
3187                        .unwrap_or("the CLI ended the stream without delivering its answer");
3188                    last_err = format!("the CLI dropped the stream ({why})");
3189                }
3190                AgentOutcome::Failed(e) => last_err = e,
3191                // The continuation was never started; the report stays lost
3192                // and the caller records that, without any handover.
3193                AgentOutcome::ResourceBlocked(e) => {
3194                    return (
3195                        seat,
3196                        None,
3197                        Some(format!("the continuation was not started: {e}")),
3198                        ContinuationRecord {
3199                            attempts: attempts.saturating_sub(1),
3200                            cumulative_wait_ms,
3201                            outcome: ContinuationOutcome::NoSession,
3202                        },
3203                    );
3204                }
3205            }
3206        }
3207    }
3208
3209    fn after_implement(&mut self) -> Result<()> {
3210        if !self.state.candidate_preservation.is_empty() {
3211            bail!("candidate preservation incomplete; judging is unsafe");
3212        }
3213        // Scan every candidate patch once the set is complete.
3214        if self.state.leaks.is_empty() {
3215            let cfg = self.state.config.blind.clone();
3216            let mut leaks = Vec::new();
3217            for c in &self.state.candidates {
3218                let Some(patch) =
3219                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3220                else {
3221                    continue;
3222                };
3223                leaks.extend(blind::scan(
3224                    &format!("candidate {} patch", c.label),
3225                    &patch,
3226                    &cfg.vendor_tokens,
3227                ));
3228            }
3229            if !leaks.is_empty() {
3230                let summary = leaks
3231                    .iter()
3232                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3233                    .collect::<Vec<_>>()
3234                    .join(", ");
3235                match cfg.on_leak {
3236                    LeakPolicy::Fail => {
3237                        self.state.status = RunStatus::Failed;
3238                        self.state
3239                            .event("blind", format!("vendor text in a patch: {summary}"));
3240                        self.state.leaks = leaks;
3241                        self.state.save()?;
3242                        self.settle_questions();
3243                        bail!(
3244                            "blind.on_leak = \"fail\" and vendor text reached a \
3245                             judged patch: {summary}"
3246                        );
3247                    }
3248                    LeakPolicy::Redact => self.state.event(
3249                        "blind",
3250                        format!("redacting vendor text for judging: {summary}"),
3251                    ),
3252                    LeakPolicy::Warn => self.state.event(
3253                        "blind",
3254                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3255                    ),
3256                }
3257                self.state.leaks = leaks;
3258            }
3259        }
3260
3261        if self.state.viable().is_empty() {
3262            if self.state.all_candidates_verified_noop() {
3263                // Every candidate agreed, with evidence the adoption guard
3264                // accepted, that nothing belongs in this worktree. That is
3265                // not the same fact as a candidate that simply failed to
3266                // write anything, and settling it as an ordinary `Failed`
3267                // (see `SCHEMA`'s doc for schema 10) is what let two of
3268                // task 391f's attempts burn a retry each re-discovering the
3269                // same already-landed fix. Terminal either way, so `judge`
3270                // must never run over an empty candidate set — unlike the
3271                // `Failed` branch below this returns `Ok`, not an error:
3272                // nothing here failed.
3273                self.state.status = RunStatus::VerifiedNoop;
3274                self.state.save()?;
3275                self.settle_questions();
3276                return Ok(());
3277            }
3278            self.state.status = RunStatus::Failed;
3279            self.state.save()?;
3280            self.settle_questions();
3281            bail!("no candidate produced a change; nothing to judge");
3282        }
3283        self.state.status = RunStatus::Judging;
3284        self.state.save()?;
3285        Ok(())
3286    }
3287
3288    // --------------------------------------------------------------- judge
3289
3290    async fn judge(&mut self) -> Result<()> {
3291        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3292        // agent files with `magi task add` name the run that paid for it. The
3293        // prompt overlay is cloned alongside it because the waves borrow it
3294        // while `self` is mutably borrowed by the node's own bookkeeping.
3295        let run_id = self.state.id.clone();
3296        let prompts = self.state.config.prompts.clone();
3297        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3298            return Ok(());
3299        }
3300        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3301        if viable.len() == 1 {
3302            // Recorded so this is a one-time event: `judgements` stays empty
3303            // either way, which without this flag is indistinguishable from
3304            // "not yet judged" on the next reentry — and status is left
3305            // untouched, so a later node's conclusion (e.g. `Blocked` after
3306            // the review budget ran out) survives a resume instead of being
3307            // clobbered back to `Judging` by this node running again.
3308            self.state.judge_skipped = true;
3309            self.state.event(
3310                "judge",
3311                format!(
3312                    "only candidate {} produced a change; judging skipped",
3313                    viable[0].label
3314                ),
3315            );
3316            self.state.save()?;
3317            return Ok(());
3318        }
3319        self.state.status = RunStatus::Judging;
3320
3321        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3322        let language = self.state.config.graph.language.clone();
3323        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3324        let sessions = self.state.config.graph.sessions;
3325        let artifacts = agent::artifacts_dir(&self.state.dir());
3326        let root = self.state.worktree_root();
3327        let base_short = short(&self.state.base_commit);
3328
3329        let mut jobs = Vec::new();
3330        let mut orders = Vec::new();
3331        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3332            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3333            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3334            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3335            let seat_key = format!("judge-{}", j + 1);
3336            let seat = self.seat(&seat_key, &spec.id);
3337            jobs.push(SeatJob {
3338                prompt: prompt::judge(
3339                    &self.state.instruction,
3340                    &views,
3341                    self.roles.judges.len(),
3342                    &base_short,
3343                    &language,
3344                ),
3345                spec,
3346                seat,
3347                cwd: root.join(format!("judge-{}", j + 1)),
3348                timeout,
3349                allow_write: false,
3350                sessions,
3351                artifacts: artifacts.clone(),
3352                stem: format!("judge-{}", j + 1),
3353                handover: None,
3354            });
3355        }
3356
3357        self.state.event(
3358            "judge",
3359            format!(
3360                "{} judges ranking {} candidates blind",
3361                jobs.len(),
3362                viable.len()
3363            ),
3364        );
3365        let labels_for_check = labels.clone();
3366        let mut quota_losses = Vec::new();
3367        let cache = self.state.config.cache_dir();
3368        let ctx = WaveCtx {
3369            carry_seats: false,
3370            pauses: Some((&self.pause, &self.interrupt)),
3371            run: &run_id,
3372            node: "judge",
3373            prompts: &prompts,
3374            cache: cache.as_deref(),
3375            round: None,
3376        };
3377        let results = ask_json_wave::<Ranking>(
3378            jobs,
3379            Arc::clone(&self.sem),
3380            self.state.config.graph.retries,
3381            &self.roles.judge_roster,
3382            &ctx,
3383            &mut quota_losses,
3384            &mut self.state,
3385            &move |r: &Ranking| r.validate(&labels_for_check),
3386        )
3387        .await;
3388        if self.state.resource_block.is_some() {
3389            return self.stop_for_resource();
3390        }
3391        self.state.quota.extend(quota_losses);
3392
3393        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3394            let agent_id = seat.agent.clone();
3395            self.state.seats.insert(seat.key.clone(), seat);
3396            let mut record = Judgement {
3397                judge: j + 1,
3398                seat: format!("judge-{}", j + 1),
3399                agent: agent_id,
3400                ranking: Vec::new(),
3401                reasons: BTreeMap::new(),
3402                confidence: None,
3403                strengths: BTreeMap::new(),
3404                order: orders[j].clone(),
3405                failed: None,
3406                duration_ms: 0,
3407            };
3408            match res {
3409                Ok((ranking, out)) => {
3410                    record.ranking = ranking.normalized();
3411                    record.reasons = ranking.reasons;
3412                    record.strengths = ranking.strengths;
3413                    record.confidence = ranking.confidence;
3414                    record.duration_ms = out.duration_ms;
3415                    self.state.event(
3416                        "judge",
3417                        format!(
3418                            "judge {} ranked {}",
3419                            j + 1,
3420                            record.ranking.iter().collect::<String>()
3421                        ),
3422                    );
3423                }
3424                Err(e) => {
3425                    record.failed = Some(e.to_string());
3426                    self.state
3427                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3428                }
3429            }
3430            self.state.judgements.push(record);
3431            self.state.save()?;
3432        }
3433        Ok(())
3434    }
3435
3436    // ---------------------------------------------------------- deliberate
3437
3438    async fn deliberate(&mut self) -> Result<()> {
3439        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3440        // agent files with `magi task add` name the run that paid for it. The
3441        // prompt overlay is cloned alongside it because the waves borrow it
3442        // while `self` is mutably borrowed by the node's own bookkeeping.
3443        let run_id = self.state.id.clone();
3444        let prompts = self.state.config.prompts.clone();
3445        if !self.state.deliberation.is_empty() {
3446            return Ok(());
3447        }
3448        let tops: Vec<char> = self
3449            .state
3450            .judgements
3451            .iter()
3452            .filter_map(|j| j.ranking.first().copied())
3453            .collect();
3454        let rounds = self.state.config.graph.deliberate_rounds;
3455        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3456            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3457                self.state.event(
3458                    "deliberate",
3459                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3460                );
3461            }
3462            self.state.status = RunStatus::Voting;
3463            self.state.save()?;
3464            return Ok(());
3465        }
3466
3467        self.state.status = RunStatus::Deliberating;
3468        self.state.event(
3469            "deliberate",
3470            format!(
3471                "split: first choices were {} — opening {rounds} round(s)",
3472                tops.iter().collect::<String>()
3473            ),
3474        );
3475
3476        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3477        let language = self.state.config.graph.language.clone();
3478        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3479        let sessions = self.state.config.graph.sessions;
3480        let artifacts = agent::artifacts_dir(&self.state.dir());
3481        let root = self.state.worktree_root();
3482        let base_short = short(&self.state.base_commit);
3483
3484        // Judges argue in sequence so that a turn can answer the one before it;
3485        // that is the difference between deliberation and three parallel
3486        // monologues.
3487        for round in 1..=rounds {
3488            let mut turns: Vec<DeliberationTurn> = Vec::new();
3489            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3490                if self.state.judgements[j].failed.is_some() {
3491                    continue;
3492                }
3493                let seat_key = format!("judge-{}", j + 1);
3494                let spec = self.occupant(&seat_key, spec);
3495                let mut seat = self.seat(&seat_key, &spec.id);
3496                let transcript = self.transcript(&turns, j);
3497                let build = |context: Option<&str>| {
3498                    prompt::deliberate(
3499                        &self.state.instruction,
3500                        context,
3501                        &transcript,
3502                        round,
3503                        rounds,
3504                        &language,
3505                    )
3506                };
3507                let block = self.candidate_block(&viable, &base_short);
3508                let full = build(Some(&block));
3509                let text = if has_context(&spec, &seat, sessions) {
3510                    build(None)
3511                } else {
3512                    full.clone()
3513                };
3514                let job = SeatJob {
3515                    spec,
3516                    seat: seat.clone(),
3517                    prompt: text,
3518                    cwd: root.join(format!("judge-{}", j + 1)),
3519                    timeout,
3520                    allow_write: false,
3521                    sessions,
3522                    artifacts: artifacts.clone(),
3523                    stem: format!("delib-{round}-judge-{}", j + 1),
3524                    handover: Some(full),
3525                };
3526                let cache = self.state.config.cache_dir();
3527                let ctx = WaveCtx {
3528                    carry_seats: false,
3529                    pauses: Some((&self.pause, &self.interrupt)),
3530                    run: &run_id,
3531                    node: "deliberate",
3532                    prompts: &prompts,
3533                    cache: cache.as_deref(),
3534                    round: None,
3535                };
3536                // A turn is never nudged (`retries` 0): a failed seat is
3537                // handed to the next roster agent, which gets the full
3538                // context. An empty answer is a turn, not a failure.
3539                let mut losses = Vec::new();
3540                let mut results = ask_wave_with::<String>(
3541                    vec![job],
3542                    Arc::clone(&self.sem),
3543                    0,
3544                    &self.roles.judge_roster,
3545                    &ctx,
3546                    &mut losses,
3547                    &mut self.state,
3548                    &|text: &str| {
3549                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3550                    },
3551                )
3552                .await;
3553                self.state.quota.extend(losses);
3554                let (updated, res, _) = results.pop().expect("one job in, one result out");
3555                seat = updated;
3556                let agent_id = seat.agent.clone();
3557                self.state.seats.insert(seat.key.clone(), seat);
3558                let body = match res {
3559                    Ok((body, _)) => body,
3560                    // Skip the seat; a CLI's raw error JSON is never read as
3561                    // this judge's position.
3562                    Err(e) => {
3563                        self.state
3564                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3565                        continue;
3566                    }
3567                };
3568                let tentative = verdict::extract_json::<Position>(&body)
3569                    .ok()
3570                    .and_then(|p| p.tentative)
3571                    .and_then(|s| s.trim().chars().next())
3572                    .map(|c| c.to_ascii_uppercase());
3573                self.state.event(
3574                    "deliberate",
3575                    format!(
3576                        "round {round}: judge {} now favours {}",
3577                        j + 1,
3578                        tentative.map_or("—".to_owned(), |c| c.to_string())
3579                    ),
3580                );
3581                turns.push(DeliberationTurn {
3582                    judge: j + 1,
3583                    agent: agent_id,
3584                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3585                    tentative,
3586                });
3587            }
3588            self.state
3589                .deliberation
3590                .push(DeliberationRound { round, turns });
3591            self.state.save()?;
3592        }
3593
3594        self.state.status = RunStatus::Voting;
3595        self.state.save()?;
3596        Ok(())
3597    }
3598
3599    // ---------------------------------------------------------------- vote
3600
3601    async fn vote(&mut self) -> Result<()> {
3602        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3603        // agent files with `magi task add` name the run that paid for it. The
3604        // prompt overlay is cloned alongside it because the waves borrow it
3605        // while `self` is mutably borrowed by the node's own bookkeeping.
3606        let run_id = self.state.id.clone();
3607        let prompts = self.state.config.prompts.clone();
3608        if !self.state.votes.is_empty() {
3609            return Ok(());
3610        }
3611        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3612        if viable.len() == 1 {
3613            return Ok(());
3614        }
3615        self.state.status = RunStatus::Voting;
3616
3617        let language = self.state.config.graph.language.clone();
3618        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3619        let sessions = self.state.config.graph.sessions;
3620        let artifacts = agent::artifacts_dir(&self.state.dir());
3621        let root = self.state.worktree_root();
3622        let base_short = short(&self.state.base_commit);
3623        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3624
3625        let mut jobs = Vec::new();
3626        let mut seats_at = Vec::new();
3627        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3628            if self
3629                .state
3630                .judgements
3631                .get(j)
3632                .is_some_and(|r| r.failed.is_some())
3633            {
3634                continue;
3635            }
3636            let seat_key = format!("judge-{}", j + 1);
3637            let spec = self.occupant(&seat_key, spec);
3638            let seat = self.seat(&seat_key, &spec.id);
3639            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3640            let text = if has_context(&spec, &seat, sessions) {
3641                prompt::final_vote(&viable, &language)
3642            } else {
3643                full.clone()
3644            };
3645            jobs.push(SeatJob {
3646                spec,
3647                seat,
3648                prompt: text,
3649                cwd: root.join(format!("judge-{}", j + 1)),
3650                timeout,
3651                allow_write: false,
3652                sessions,
3653                artifacts: artifacts.clone(),
3654                stem: format!("vote-judge-{}", j + 1),
3655                handover: Some(full),
3656            });
3657            seats_at.push(j);
3658        }
3659
3660        self.state.event(
3661            "vote",
3662            format!(
3663                "collecting {} final votes one by one, privately",
3664                jobs.len()
3665            ),
3666        );
3667        let allowed = viable.clone();
3668        let mut quota_losses = Vec::new();
3669        let cache = self.state.config.cache_dir();
3670        let ctx = WaveCtx {
3671            carry_seats: false,
3672            pauses: Some((&self.pause, &self.interrupt)),
3673            run: &run_id,
3674            node: "vote",
3675            prompts: &prompts,
3676            cache: cache.as_deref(),
3677            round: None,
3678        };
3679        let results = ask_json_wave::<FinalVote>(
3680            jobs,
3681            Arc::clone(&self.sem),
3682            self.state.config.graph.retries,
3683            &self.roles.judge_roster,
3684            &ctx,
3685            &mut quota_losses,
3686            &mut self.state,
3687            &move |v: &FinalVote| match v.label() {
3688                Some(c) if allowed.contains(&c) => Ok(()),
3689                other => bail!("vote {other:?} is not one of {allowed:?}"),
3690            },
3691        )
3692        .await;
3693        if self.state.resource_block.is_some() {
3694            return self.stop_for_resource();
3695        }
3696        self.state.quota.extend(quota_losses);
3697
3698        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3699            let agent_id = seat.agent.clone();
3700            self.state.seats.insert(seat.key.clone(), seat);
3701            let initial = self
3702                .state
3703                .judgements
3704                .get(j)
3705                .and_then(|r| r.ranking.first().copied());
3706            let mut record = VoteRecord {
3707                judge: j + 1,
3708                agent: agent_id,
3709                vote: None,
3710                reason: String::new(),
3711                changed: false,
3712            };
3713            match res {
3714                Ok((v, _)) => {
3715                    record.vote = v.label();
3716                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3717                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3718                    self.state.event(
3719                        "vote",
3720                        format!(
3721                            "judge {} voted {}{}",
3722                            j + 1,
3723                            record.vote.unwrap_or('?'),
3724                            if record.changed { " (changed)" } else { "" }
3725                        ),
3726                    );
3727                }
3728                Err(e) => {
3729                    self.state
3730                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3731                }
3732            }
3733            self.state.votes.push(record);
3734            self.state.save()?;
3735        }
3736        Ok(())
3737    }
3738
3739    // --------------------------------------------------------------- tally
3740
3741    fn tally(&mut self) -> Result<()> {
3742        if self.state.tally.is_some() {
3743            return Ok(());
3744        }
3745        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3746        let tops: Vec<char> = self
3747            .state
3748            .judgements
3749            .iter()
3750            .filter_map(|j| j.ranking.first().copied())
3751            .collect();
3752        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3753
3754        // A judge whose private vote failed still counted once, in the initial
3755        // ranking; using it beats discarding a whole seat.
3756        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3757        let mut cast: Vec<char> = Vec::new();
3758        for (i, j) in self.state.judgements.iter().enumerate() {
3759            let vote = self
3760                .state
3761                .votes
3762                .iter()
3763                .find(|v| v.judge == i + 1)
3764                .and_then(|v| v.vote)
3765                .or_else(|| j.ranking.first().copied());
3766            if let Some(v) = vote {
3767                *first_choice.entry(v).or_insert(0) += 1;
3768                cast.push(v);
3769            }
3770        }
3771
3772        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3773        for j in &self.state.judgements {
3774            let n = j.ranking.len();
3775            for (pos, label) in j.ranking.iter().enumerate() {
3776                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3777            }
3778        }
3779
3780        let best = first_choice.values().copied().max().unwrap_or(0);
3781        let mut leaders: Vec<char> = first_choice
3782            .iter()
3783            .filter(|(_, v)| **v == best)
3784            .map(|(k, _)| *k)
3785            .collect();
3786        let mut tie_break = None;
3787        if leaders.len() > 1 {
3788            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3789            let borda_leaders: Vec<char> = leaders
3790                .iter()
3791                .copied()
3792                .filter(|l| borda[l] == top_borda)
3793                .collect();
3794            tie_break = Some(if borda_leaders.len() == 1 {
3795                format!(
3796                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3797                    leaders.len()
3798                )
3799            } else {
3800                format!(
3801                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3802                    leaders.len()
3803                )
3804            });
3805            leaders = borda_leaders;
3806            leaders.sort_unstable();
3807        }
3808        let winner = *leaders
3809            .first()
3810            .or(viable.first())
3811            .context("no candidate to declare a winner from")?;
3812
3813        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3814        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3815        let deliberated = !self.state.deliberation.is_empty();
3816
3817        // Whose verdict is this? A rate-limited seat is absent even if it
3818        // ranked before the limit hit, so presence is measured against the
3819        // recorded losses, not just "did a ranking ever appear".
3820        let quota_seats: std::collections::BTreeSet<&str> =
3821            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3822        let mut present = 0usize;
3823        for (i, j) in self.state.judgements.iter().enumerate() {
3824            if quota_seats.contains(j.seat.as_str()) {
3825                continue;
3826            }
3827            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3828            let voted = self
3829                .state
3830                .votes
3831                .iter()
3832                .any(|v| v.judge == i + 1 && v.vote.is_some());
3833            if ranked || voted {
3834                present += 1;
3835            }
3836        }
3837        // Strict majority of the configured panel. A bare majority is real
3838        // signal we can act on, while a minority verdict must never stand in
3839        // for a healthy one. A one-candidate run needs no panel at all, and
3840        // `judges` stays `0` rather than the roster size a panel that never
3841        // sat would otherwise be credited with.
3842        let needs_quorum = viable.len() > 1;
3843        let judges_total = if needs_quorum {
3844            self.roles.judges.len()
3845        } else {
3846            0
3847        };
3848        let quorum = if needs_quorum {
3849            judges_total / 2 + 1
3850        } else {
3851            0
3852        };
3853        let met_quorum = !needs_quorum || present >= quorum;
3854        let uncontested = (!needs_quorum).then(|| {
3855            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3856        });
3857
3858        self.state.event(
3859            "tally",
3860            match &uncontested {
3861                Some(reason) => format!("winner {winner} — {reason}"),
3862                None => format!(
3863                    "winner {winner} — votes {} | initial {} | {} changed | \
3864                     {present}/{judges_total} judges{}",
3865                    first_choice
3866                        .iter()
3867                        .map(|(k, v)| format!("{k}:{v}"))
3868                        .collect::<Vec<_>>()
3869                        .join(" "),
3870                    if unanimous_initial {
3871                        "unanimous"
3872                    } else {
3873                        "split"
3874                    },
3875                    changed_votes,
3876                    if met_quorum {
3877                        String::new()
3878                    } else {
3879                        format!(" — below quorum ({quorum} required)")
3880                    },
3881                ),
3882            },
3883        );
3884        if !met_quorum {
3885            self.state.event(
3886                "stall",
3887                format!(
3888                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3889                     the run stops here, resumable"
3890                ),
3891            );
3892        }
3893        self.state.tally = Some(Tally {
3894            first_choice,
3895            borda,
3896            winner,
3897            rankings: tops.len(),
3898            unanimous_initial,
3899            deliberated,
3900            changed_votes,
3901            unanimous_final,
3902            tie_break,
3903            judges: judges_total,
3904            present,
3905            quorum,
3906            met_quorum,
3907            uncontested,
3908        });
3909        self.state.status = if met_quorum {
3910            RunStatus::Reviewing
3911        } else {
3912            RunStatus::Stalled
3913        };
3914        self.state.save()?;
3915        Ok(())
3916    }
3917
3918    // ------------------------------------------------------------- recover
3919
3920    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3921    /// resumed toward completion once the transient cause clears.
3922    ///
3923    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3924    /// it toward the quorum, which is exactly the set of seats whose absence
3925    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3926    /// not depend on which node happened to hit the limit), or an ordinary
3927    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3928    /// seat is never disturbed.
3929    ///
3930    /// A seat that now answers with a usable ranking is "recovered": its
3931    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3932    /// `tally` counts it present again), and its vote re-collected. A seat that
3933    /// still fails keeps its loss and stays absent.
3934    ///
3935    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3936    /// to review/gate/merge), `false` when it is still below quorum (the run
3937    /// stays `Stalled`, still resumable for a later retry).
3938    #[allow(clippy::too_many_lines)]
3939    async fn recover_stall(&mut self) -> Result<bool> {
3940        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3941        // agent files with `magi task add` name the run that paid for it. The
3942        // prompt overlay is cloned alongside it because the waves borrow it
3943        // while `self` is mutably borrowed by the node's own bookkeeping.
3944        let run_id = self.state.id.clone();
3945        let prompts = self.state.config.prompts.clone();
3946        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3947        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3948        // is what keeps a non-quota collapse — or a quota loss recorded at the
3949        // deliberate node — from being a permanent dead-end on `--resume`.
3950        let quota_seats: BTreeSet<&str> =
3951            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3952        let absent: Vec<String> = self
3953            .state
3954            .judgements
3955            .iter()
3956            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3957            .map(|j| j.seat.clone())
3958            .collect();
3959        if absent.is_empty() {
3960            return Ok(false);
3961        }
3962        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3963        if viable.len() <= 1 {
3964            return Ok(false);
3965        }
3966        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3967        let language = self.state.config.graph.language.clone();
3968        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3969        let sessions = self.state.config.graph.sessions;
3970        let artifacts = agent::artifacts_dir(&self.state.dir());
3971        let root = self.state.worktree_root();
3972        let base_short = short(&self.state.base_commit);
3973        let candidates: Vec<Candidate> = viable.clone();
3974
3975        // Map each absent seat key to its 0-based position in `roles.judges`.
3976        let mut positions: Vec<usize> = absent
3977            .iter()
3978            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3979            .collect();
3980        if positions.is_empty() {
3981            return Ok(false);
3982        }
3983        positions.sort_unstable();
3984        positions.dedup();
3985
3986        // Re-rank the lost seats, one blind prompt each.
3987        let mut judge_jobs = Vec::new();
3988        for &j in &positions {
3989            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3990            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3991            let seat_key = format!("judge-{}", j + 1);
3992            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3993            let seat = self.seat(&seat_key, &spec.id);
3994            judge_jobs.push(SeatJob {
3995                spec,
3996                seat,
3997                prompt: prompt::judge(
3998                    &self.state.instruction,
3999                    &views,
4000                    self.roles.judges.len(),
4001                    &base_short,
4002                    &language,
4003                ),
4004                cwd: root.join(seat_key),
4005                timeout,
4006                allow_write: false,
4007                sessions,
4008                artifacts: artifacts.clone(),
4009                stem: format!("judge-{}-recover", j + 1),
4010                handover: None,
4011            });
4012        }
4013
4014        let labels_for_check = labels.clone();
4015        let mut judge_losses = Vec::new();
4016        let retries = self.state.config.graph.retries;
4017        let cache = self.state.config.cache_dir();
4018        let ctx = WaveCtx {
4019            carry_seats: false,
4020            pauses: Some((&self.pause, &self.interrupt)),
4021            run: &run_id,
4022            node: "judge",
4023            prompts: &prompts,
4024            cache: cache.as_deref(),
4025            round: None,
4026        };
4027        let results = ask_json_wave::<Ranking>(
4028            judge_jobs,
4029            Arc::clone(&self.sem),
4030            retries,
4031            &self.roles.judge_roster,
4032            &ctx,
4033            &mut judge_losses,
4034            &mut self.state,
4035            &move |r: &Ranking| r.validate(&labels_for_check),
4036        )
4037        .await;
4038        if self.state.resource_block.is_some() {
4039            self.stop_for_resource()?;
4040            return Ok(false);
4041        }
4042
4043        // Refresh the judgement of every seat that ranked again.
4044        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4045        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4046            let agent_id = seat.agent.clone();
4047            self.state.seats.insert(seat.key.clone(), seat);
4048            let record = &mut self.state.judgements[j];
4049            match res {
4050                Ok((ranking, out)) => {
4051                    record.agent = agent_id;
4052                    record.ranking = ranking.normalized();
4053                    record.reasons = ranking.reasons;
4054                    record.strengths = ranking.strengths;
4055                    record.confidence = ranking.confidence;
4056                    record.failed = None;
4057                    record.duration_ms = out.duration_ms;
4058                    recovered.insert(j);
4059                    self.state.event(
4060                        "recover",
4061                        format!("judge {} ranked again after the limit", j + 1),
4062                    );
4063                }
4064                Err(e) => {
4065                    self.state
4066                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4067                }
4068            }
4069        }
4070
4071        // Re-ask the votes of the seats that recovered a ranking.
4072        let mut vote_jobs = Vec::new();
4073        let mut vote_pos: Vec<usize> = Vec::new();
4074        for &j in &recovered {
4075            let seat_key = format!("judge-{}", j + 1);
4076            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4077            let seat = self.seat(&seat_key, &spec.id);
4078            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4079            let text = if has_context(&spec, &seat, sessions) {
4080                prompt::final_vote(&labels, &language)
4081            } else {
4082                full.clone()
4083            };
4084            vote_jobs.push(SeatJob {
4085                spec,
4086                seat,
4087                prompt: text,
4088                cwd: root.join(seat_key),
4089                timeout,
4090                allow_write: false,
4091                sessions,
4092                artifacts: artifacts.clone(),
4093                stem: format!("vote-judge-{}-recover", j + 1),
4094                handover: Some(full),
4095            });
4096            vote_pos.push(j);
4097        }
4098        let allowed = labels.clone();
4099        let mut vote_losses = Vec::new();
4100        let vote_retries = self.state.config.graph.retries;
4101        let vote_cache = self.state.config.cache_dir();
4102        let ctx = WaveCtx {
4103            carry_seats: false,
4104            pauses: Some((&self.pause, &self.interrupt)),
4105            run: &run_id,
4106            node: "vote",
4107            prompts: &prompts,
4108            cache: vote_cache.as_deref(),
4109            round: None,
4110        };
4111        let votes = ask_json_wave::<FinalVote>(
4112            vote_jobs,
4113            Arc::clone(&self.sem),
4114            vote_retries,
4115            &self.roles.judge_roster,
4116            &ctx,
4117            &mut vote_losses,
4118            &mut self.state,
4119            &move |v: &FinalVote| match v.label() {
4120                Some(c) if allowed.contains(&c) => Ok(()),
4121                other => bail!("vote {other:?} is not one of {allowed:?}"),
4122            },
4123        )
4124        .await;
4125        if self.state.resource_block.is_some() {
4126            self.stop_for_resource()?;
4127            return Ok(false);
4128        }
4129        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4130            let agent_id = seat.agent.clone();
4131            self.state.seats.insert(seat.key.clone(), seat);
4132            match res {
4133                Ok((v, _)) => {
4134                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4135                        rec.vote = v.label();
4136                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4137                    } else {
4138                        self.state.votes.push(VoteRecord {
4139                            judge: j + 1,
4140                            agent: agent_id,
4141                            vote: v.label(),
4142                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4143                            changed: false,
4144                        });
4145                    }
4146                    self.state.event(
4147                        "recover",
4148                        format!("judge {} voted again after the limit", j + 1),
4149                    );
4150                }
4151                Err(e) => {
4152                    self.state
4153                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4154                }
4155            }
4156        }
4157
4158        // A seat that ranked again is present even if its re-vote failed —
4159        // `tally` falls back to the initial ranking's first choice — so clear
4160        // its quota loss. Seats that still fail keep theirs and stay absent.
4161        let recovered_keys: BTreeSet<String> = recovered
4162            .iter()
4163            .map(|&j| format!("judge-{}", j + 1))
4164            .collect();
4165        self.state
4166            .quota
4167            .retain(|q| !recovered_keys.contains(&q.seat));
4168        // A seat that hit the limit again is a fresh loss, not the old one:
4169        // replace the stale entry so the history stays one-per-seat and the
4170        // daemon can tell this attempt's loss from a previous session's.
4171        for loss in judge_losses.into_iter().chain(vote_losses) {
4172            if recovered_keys.contains(&loss.seat) {
4173                continue;
4174            }
4175            self.state.quota.retain(|q| q.seat != loss.seat);
4176            self.state.quota.push(loss);
4177        }
4178
4179        // Recompute the verdict from the refreshed panel.
4180        self.state.tally = None;
4181        self.tally()?;
4182        Ok(self
4183            .state
4184            .tally
4185            .as_ref()
4186            .map(|t| t.met_quorum)
4187            .unwrap_or(false))
4188    }
4189
4190    // ---------------------------------------------------------------- graft
4191
4192    /// Port the strengths the judges attributed to the losing candidates onto
4193    /// the winner, once, between the tally and the review loop.
4194    ///
4195    /// Best-effort by construction: it returns nothing and fails nothing. A
4196    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4197    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4198    /// the run carries on as if the phase were off. See [`graft`] for the
4199    /// decisions and `AGENTS.md` for the invariants.
4200    ///
4201    /// The one exception is a rollback that could not be confirmed: the
4202    /// rejected commit may still be the winner's tip, so the run is blocked
4203    /// (resumable; the record stays `Running`, which makes a resume retry the
4204    /// restore) instead of reviewing, gating and merging it.
4205    async fn graft(&mut self) -> Result<()> {
4206        let Err(e) = self.graft_inner().await else {
4207            return Ok(());
4208        };
4209        if e.is::<crate::repo_lock::ResourceBlocked>() {
4210            return Err(e);
4211        }
4212        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4213            self.state.status = RunStatus::Blocked;
4214            self.state.event("graft", format!("blocked: {e:#}"));
4215            self.state.save()?;
4216            return Err(e);
4217        }
4218        // An error after the `Running` record was saved may have left the
4219        // fixer's work in the tree: the record must not stay unsettled
4220        // (a later resume would reset past the review fixes), so confirm the
4221        // restore now, or block.
4222        if let Some(rec) = self.state.graft.clone()
4223            && !rec.outcome.settled()
4224        {
4225            if let Some(winner) = self.state.winner().cloned() {
4226                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4227                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4228                        return Err(re);
4229                    }
4230                    self.state.status = RunStatus::Blocked;
4231                    self.state.event("graft", format!("blocked: {re:#}"));
4232                    self.state.save()?;
4233                    return Err(re);
4234                }
4235            }
4236            if let Some(g) = self.state.graft.as_mut() {
4237                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4238            }
4239        }
4240        self.state.event("graft", format!("skipped: {e:#}"));
4241        if let Err(e) = self.state.save() {
4242            tracing::warn!("save after graft: {e:#}");
4243        }
4244        Ok(())
4245    }
4246
4247    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4248    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4249    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4250        let _coordination = self.coordinate(wt).await?;
4251        crate::recovery::restore_repair(wt, pre)
4252            .await
4253            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4254    }
4255
4256    /// A skip is a run event, recorded once however often the node is re-entered.
4257    fn graft_skipped(&mut self, why: &str) {
4258        let message = format!("skipped: {why}");
4259        if !self
4260            .state
4261            .events
4262            .iter()
4263            .any(|e| e.node == "graft" && e.message == message)
4264        {
4265            self.state.event("graft", message);
4266        }
4267    }
4268
4269    async fn graft_inner(&mut self) -> Result<()> {
4270        // A round that was started and never finished (crash, kill): the
4271        // fixer may have left anything in the tree, so put the winner back.
4272        if let Some(rec) = self.state.graft.clone()
4273            && !rec.outcome.settled()
4274        {
4275            // Reviews exist only if the graft ended without its record being
4276            // settled (a swallowed error): the tree has moved on since, and
4277            // a reset to `pre` would discard the review fixes.
4278            if self.state.reviews.is_empty()
4279                && let Some(winner) = self.state.winner().cloned()
4280            {
4281                self.graft_restore(&winner.worktree, &rec.pre).await?;
4282            }
4283            if rec.outcome == GraftOutcome::WaitingForRestore {
4284                self.state.graft = None;
4285                self.state.save()?;
4286            } else {
4287                if let Some(g) = self.state.graft.as_mut() {
4288                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4289                }
4290                self.state.event(
4291                    "graft",
4292                    "rolled back: the previous round was interrupted; the winner is untouched",
4293                );
4294                self.state.save()?;
4295                return Ok(());
4296            }
4297        }
4298        let plan = match graft::plan(&self.state) {
4299            Ok(plan) => plan,
4300            Err(skip) => {
4301                if skip != graft::Skip::AlreadyDone {
4302                    self.graft_skipped(skip.reason());
4303                    self.state.save()?;
4304                }
4305                return Ok(());
4306            }
4307        };
4308        let Some(winner) = self.state.winner().cloned() else {
4309            return Ok(());
4310        };
4311        let wt = winner.worktree.clone();
4312        if !git::is_clean(&wt).await? {
4313            self.graft_skipped("the winner's worktree has uncommitted changes");
4314            self.state.save()?;
4315            return Ok(());
4316        }
4317        let pre = git::rev_parse(&wt, "HEAD").await?;
4318        // Saved before the fixer is called: a stop from here on is repaired
4319        // by the branch above on the next entry.
4320        self.state.graft = Some(GraftRecord {
4321            pre: pre.clone(),
4322            post: None,
4323            lines: None,
4324            limit: None,
4325            agent: None,
4326            outcome: GraftOutcome::Running,
4327            requested: graft::requested(&plan),
4328            adoption: None,
4329            adoption_error: None,
4330        });
4331        self.state.event(
4332            "graft",
4333            format!(
4334                "porting strengths of candidate(s) {} onto {}",
4335                plan.strengths
4336                    .iter()
4337                    .map(|(l, _, _)| l.to_string())
4338                    .collect::<Vec<_>>()
4339                    .join(", "),
4340                plan.winner
4341            ),
4342        );
4343        self.state.save()?;
4344
4345        let fix_prompt = prompt::graft(
4346            &self.state.instruction,
4347            plan.winner,
4348            &plan.strengths,
4349            &self.state.config.graph.language,
4350        );
4351        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4352        let sessions = self.state.config.graph.sessions;
4353        let artifacts = agent::artifacts_dir(&self.state.dir());
4354        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4355        let (_, seat, out) = self
4356            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4357                SeatJob {
4358                    prompt: fix_prompt.clone(),
4359                    spec,
4360                    seat,
4361                    cwd: wt.clone(),
4362                    timeout,
4363                    allow_write: true,
4364                    sessions,
4365                    artifacts: artifacts.clone(),
4366                    stem: "graft".to_owned(),
4367                    handover: None,
4368                }
4369            })
4370            .await;
4371        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4372            // Preserve the need to restore earlier fallback edits without
4373            // confusing an unstarted seat with an interrupted invocation.
4374            self.state.graft.as_mut().expect("graft record").outcome =
4375                GraftOutcome::WaitingForRestore;
4376            return self.stop_for_resource();
4377        }
4378        let mut why: Option<String> = None;
4379        match &out {
4380            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4381                why = Some("the fixer produced nothing usable".to_owned());
4382            }
4383            AgentOutcome::Ok(_) => {}
4384            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4385            AgentOutcome::Quota(o) => {
4386                self.state.quota.push(QuotaLoss {
4387                    seat: seat.key.clone(),
4388                    node: "graft".to_owned(),
4389                    at: Timestamp::now(),
4390                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4391                });
4392                why = Some("rate limited (quota)".to_owned());
4393            }
4394            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4395                why = Some(format!(
4396                    "the fixer failed: {}",
4397                    e.lines()
4398                        .next()
4399                        .unwrap_or("")
4400                        .chars()
4401                        .take(160)
4402                        .collect::<String>()
4403                ));
4404            }
4405        }
4406        let agent_id = seat.agent.clone();
4407        self.state.seats.insert(seat.key.clone(), seat);
4408
4409        let mut post = None;
4410        let mut lines = None;
4411        let _coordination = self.coordinate(&wt).await?;
4412        if why.is_none() {
4413            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4414            if let Ok(r) = &rescued {
4415                self.state.note_withheld("graft", &r.withheld);
4416            }
4417            let head = git::rev_parse(&wt, "HEAD").await?;
4418            // What is measured must be what gets reviewed and gated: a failed
4419            // rescue, or anything still uncommitted (withheld lockfiles
4420            // aside, which are never part of the tree), rejects the graft.
4421            let dirty = match &rescued {
4422                Ok(r) => git::status_porcelain(&wt)
4423                    .await
4424                    .map(|st| {
4425                        st.lines().any(|l| {
4426                            !r.withheld
4427                                .iter()
4428                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4429                        })
4430                    })
4431                    .unwrap_or(true),
4432                Err(_) => true,
4433            };
4434            if dirty {
4435                why = Some("the fixer left changes that could not be committed".to_owned());
4436            } else if head == pre {
4437                why = Some("the fixer changed nothing".to_owned());
4438            } else {
4439                lines = git::changed_lines(&wt, &pre, &head).await;
4440                post = Some(head);
4441            }
4442        }
4443        let (adoption, adoption_error) = match &out {
4444            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4445            _ => (None, None),
4446        };
4447        let requested = graft::requested(&plan);
4448        let outcome = match why {
4449            None => {
4450                self.state.event(
4451                    "graft",
4452                    format!(
4453                        "applied: {}",
4454                        lines.map_or_else(
4455                            || "changed lines unavailable".to_owned(),
4456                            |n| format!("{n} changed lines")
4457                        )
4458                    ),
4459                );
4460                GraftOutcome::Applied
4461            }
4462            Some(why) => {
4463                // Not settled until the restore is confirmed: the record
4464                // stays `Running`, so a resume retries it.
4465                crate::recovery::restore_repair(&wt, &pre)
4466                    .await
4467                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4468                self.state.event(
4469                    "graft",
4470                    format!("rolled back: {why}; the winner is untouched"),
4471                );
4472                GraftOutcome::RolledBack(why)
4473            }
4474        };
4475        self.state.graft = Some(GraftRecord {
4476            pre,
4477            post,
4478            lines,
4479            limit: None,
4480            agent: Some(agent_id),
4481            outcome,
4482            requested,
4483            adoption,
4484            adoption_error,
4485        });
4486        self.state.save()?;
4487        Ok(())
4488    }
4489
4490    // ----------------------------------------------------------------- fold
4491
4492    async fn fold_losers(&mut self) -> Result<()> {
4493        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4494            return Ok(());
4495        };
4496        if !self.state.candidate_preservation.is_empty() {
4497            bail!("candidate preservation incomplete; refusing cleanup");
4498        }
4499        let repo = self.state.repo.clone();
4500        let _coordination = self.coordinate(&repo).await?;
4501        let mut folded = Vec::new();
4502        for i in 0..self.state.candidates.len() {
4503            let c = &self.state.candidates[i];
4504            if c.label == winner || c.folded {
4505                continue;
4506            }
4507            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4508            remove_preserved_worktree(&repo, &wt).await?;
4509            if git::branch_exists(&repo, &branch).await?
4510                && !git::branch_delete(&repo, &branch).await?
4511            {
4512                bail!("candidate branch removal refused");
4513            }
4514            self.state.candidates[i].folded = true;
4515            self.state.save()?;
4516            folded.push(label.to_string());
4517        }
4518        // The judges are finished; their checkouts are pure cost from here.
4519        let root = self.state.worktree_root();
4520        for j in 1..=self.roles.judges.len() {
4521            let wt = root.join(format!("judge-{j}"));
4522            if wt.exists() {
4523                remove_preserved_worktree(&repo, &wt).await?;
4524            }
4525        }
4526        // The design-deliberation stage is finished by the time a tally
4527        // exists — same reasoning as the judges above.
4528        if self.state.config.graph.advise {
4529            for k in 1..=self.state.config.graph.advisors {
4530                let wt = root.join(format!("advisor-{k}"));
4531                if wt.exists() {
4532                    remove_preserved_worktree(&repo, &wt).await?;
4533                }
4534            }
4535        }
4536        if !folded.is_empty() {
4537            self.state
4538                .event("fold", format!("folded candidates {}", folded.join(", ")));
4539            self.state.save()?;
4540        }
4541        Ok(())
4542    }
4543
4544    // ------------------------------------------------------------ base sync
4545
4546    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4547    /// anything verifies it.
4548    ///
4549    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4550    /// read whatever is checked out in the winner's worktree. Left alone that
4551    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4552    /// it when the run *branched* - and a run takes long enough that the base
4553    /// has usually moved by the time it gets here. A gate that ran there
4554    /// answers "green on the commit this run started from", not "green on
4555    /// what is about to land", and the difference showed up three times in
4556    /// one day as a green run whose merge would have reverted a file another
4557    /// pull request had already landed.
4558    ///
4559    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4560    /// `land::Step::Rebase` calls, rather than a second implementation of the
4561    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4562    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4563    /// doc warns about ("two answers to a question nobody notices until a
4564    /// diff is wrong").
4565    ///
4566    /// A conflict is not the end of the road: the standing rebase is handed
4567    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4568    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4569    /// land). Once it finishes, review and the gate run as usual on the
4570    /// rebased tree, which is where a breakage the new base caused is caught
4571    /// by the ordinary gate-fix round. magi resolves nothing itself.
4572    ///
4573    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4574    /// `state.base_sync.attempts`, is how many times the base is *rebased
4575    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4576    /// *conflict* was given to a fixer. When the fixer cannot finish the
4577    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4578    /// what was tried (rounds spent, paths still conflicted) and the branch
4579    /// and worktree stay exactly as they were - untouched, for a person to
4580    /// look at - which is also what makes re-entering this function
4581    /// afterwards a no-op instead of a second attempt at the same wall. A
4582    /// push failure ends the same way.
4583    async fn sync_to_base(&mut self) -> Result<()> {
4584        if self.state.status == RunStatus::AlreadyInBase {
4585            return Ok(());
4586        }
4587        let conflicted = self
4588            .state
4589            .base_sync
4590            .as_ref()
4591            .is_some_and(|s| s.conflict.is_some());
4592        let Some(winner) = self.state.winner().cloned() else {
4593            return Ok(());
4594        };
4595
4596        let repo = self.state.repo.clone();
4597        let remote = self.state.config.merge.remote.clone();
4598        let base_branch = self.state.base_branch.clone();
4599        let tracking = format!("{remote}/{base_branch}");
4600
4601        git::fetch(&repo, &remote, &base_branch).await.ok();
4602        // No network, or the remote never had this branch: the run already
4603        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4604        // otherwise), and one that got this far is not blocked by a fetch
4605        // that fails now. It just has no newer tip to compare against.
4606        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4607            return Ok(());
4608        };
4609
4610        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4611        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4612        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4613
4614        // Before any rebase, and before a recorded conflict is honoured: a
4615        // branch whose change reached the base under other commit ids has
4616        // nothing to rebase and nothing to conflict with, and a run that
4617        // already stopped on that phantom conflict recovers here on resume.
4618        // `behind == 0` with head == tip is a branch the base has since taken
4619        // in whole, whether or not a conflict was ever recorded: the ancestry
4620        // proof must still run (`classify` ignores a head still on the start
4621        // commit).
4622        if (behind > 0 || conflicted || head == tip)
4623            && self
4624                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4625                .await?
4626        {
4627            return Ok(());
4628        }
4629        if conflicted {
4630            return Ok(());
4631        }
4632
4633        if behind == 0 {
4634            // A fixer-finished rebase moves the branch ref before the
4635            // winner's worktree is told (`sync_to_head` below). A run that
4636            // died in between resumes here with `behind == 0` and a tree still
4637            // holding the pre-rebase files, which review and the gate would
4638            // then read. That state is exactly: HEAD moved off the tip the
4639            // rebase started from, yet the tree is still identical to that
4640            // tip. A tree with edits of its own differs from it, so nothing
4641            // is thrown away.
4642            if let Some(from) = self
4643                .state
4644                .rebase_fixes
4645                .iter()
4646                .rev()
4647                .find_map(|r| r.from.clone())
4648                && from != head
4649                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4650                    .await
4651                    .is_ok_and(|o| o.ok())
4652            {
4653                git::sync_to_head(&winner.worktree).await?;
4654            }
4655            // An earlier attempt may have rebased the branch locally and died
4656            // before pushing it (only the fresh-rebase arm below pushes).
4657            // Publish it now, so the plain push at PR time is not refused as
4658            // a non-fast-forward. A run already holding a recorded conflict
4659            // never reaches here; that case is out of scope.
4660            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4661            if let Some(why) = &conflict {
4662                self.state.status = RunStatus::Blocked;
4663                self.state.event("land", why.clone());
4664            }
4665            self.state.base_sync = Some(BaseSync {
4666                tip,
4667                behind: 0,
4668                attempts,
4669                conflict,
4670                already_in: None,
4671            });
4672            self.state.save()?;
4673            return Ok(());
4674        }
4675
4676        if attempts >= BASE_SYNC_ROUNDS {
4677            let why = format!(
4678                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4679                 rebase(s); rebasing again would only race it",
4680                winner.branch
4681            );
4682            self.state.status = RunStatus::Blocked;
4683            self.state.base_sync = Some(BaseSync {
4684                tip,
4685                behind,
4686                attempts,
4687                conflict: Some(why.clone()),
4688                already_in: None,
4689            });
4690            self.state.event("land", why);
4691            self.state.save()?;
4692            return Ok(());
4693        }
4694
4695        self.state.event(
4696            "land",
4697            format!(
4698                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4699                winner.branch
4700            ),
4701        );
4702        self.state.save()?;
4703
4704        // The remote's copy of the branch, read now and only if the fetch
4705        // really succeeded (a stale tracking ref must never pin a lease). It is
4706        // pushed over after a rebase only when it is a commit this branch
4707        // already contains, by ancestry or by patch (an earlier rebase of ours
4708        // that never reached the remote): anything else is somebody else's work.
4709        let branch_tracking = format!("{remote}/{}", winner.branch);
4710        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4711        let remote_tip = if self.state.task_pr_branch().is_none()
4712            && matches!(&fetched_branch, Ok(o) if o.ok())
4713        {
4714            git::rev_parse(&repo, &branch_tracking).await.ok()
4715        } else {
4716            None
4717        };
4718        // A remote tip this branch does not contain is somebody else's work:
4719        // rebasing would leave a local tip that can never be pushed. Stop
4720        // before touching anything and say so.
4721        if let Some(theirs) = &remote_tip
4722            && !git::is_ancestor(&repo, theirs, &head).await
4723            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4724                .await
4725                .is_ok_and(|missing| missing.is_empty())
4726        {
4727            let why = format!(
4728                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4729                 them",
4730                short(theirs),
4731                winner.branch
4732            );
4733            self.state.status = RunStatus::Blocked;
4734            self.state.base_sync = Some(BaseSync {
4735                tip,
4736                behind,
4737                attempts,
4738                conflict: Some(why.clone()),
4739                already_in: None,
4740            });
4741            self.state.event("land", why);
4742            self.state.save()?;
4743            return Ok(());
4744        }
4745
4746        let scratch = self.state.dir().join("base-sync");
4747        let rebased = match crate::rebase::rebase_with_fixer(
4748            &mut self.state,
4749            &scratch,
4750            &winner.branch,
4751            &tracking,
4752        )
4753        .await
4754        {
4755            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4756            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4757            Err(e) => Err(e),
4758        };
4759        let attempts = attempts + 1;
4760        match rebased {
4761            Ok(None) => {
4762                // The branch ref moved, but a worktree that already had it
4763                // checked out (the winner's) was not told; sync its index and
4764                // files before anything reads them.
4765                git::sync_to_head(&winner.worktree).await?;
4766                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4767                let mut conflict = None;
4768                if let Some(pinned) = &remote_tip {
4769                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4770                    match pushed {
4771                        Ok(o) if o.ok() => self.state.event(
4772                            "land",
4773                            format!("pushed rebased {} to {remote}", winner.branch),
4774                        ),
4775                        Ok(o) => {
4776                            conflict = Some(format!(
4777                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4778                                winner.branch,
4779                                short(pinned),
4780                                o.stderr.chars().take(600).collect::<String>()
4781                            ));
4782                        }
4783                        Err(e) => {
4784                            conflict = Some(format!(
4785                                "rebased {} locally but could not push it: {e:#}",
4786                                winner.branch
4787                            ));
4788                        }
4789                    }
4790                }
4791                if let Some(why) = &conflict {
4792                    self.state.status = RunStatus::Blocked;
4793                    self.state.event("land", why.clone());
4794                }
4795                self.state.base_sync = Some(BaseSync {
4796                    tip: tip.clone(),
4797                    behind: 0,
4798                    attempts,
4799                    conflict,
4800                    already_in: None,
4801                });
4802                self.state
4803                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4804            }
4805            Ok(Some(conflict)) => {
4806                let why = format!(
4807                    "{} conflicts with {tracking} and did not rebase: {}",
4808                    winner.branch,
4809                    conflict.chars().take(600).collect::<String>()
4810                );
4811                self.state.status = RunStatus::Blocked;
4812                self.state.base_sync = Some(BaseSync {
4813                    tip,
4814                    behind,
4815                    attempts,
4816                    conflict: Some(why.clone()),
4817                    already_in: None,
4818                });
4819                self.state.event("land", why);
4820            }
4821            // A refused Git write or a busy repository: nothing is wrong with
4822            // the branch. The scratch rebase stays standing and the run
4823            // resumes into it (`base_sync` must not claim a conflict).
4824            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4825                self.state.save()?;
4826                return Ok(());
4827            }
4828            Err(e) => {
4829                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4830                self.state.status = RunStatus::Blocked;
4831                self.state.base_sync = Some(BaseSync {
4832                    tip,
4833                    behind,
4834                    attempts,
4835                    conflict: Some(why.clone()),
4836                    already_in: None,
4837                });
4838                self.state.event("land", why);
4839            }
4840        }
4841        self.state.save()?;
4842        Ok(())
4843    }
4844
4845    /// Push a branch an earlier attempt rebased locally but never published,
4846    /// pinned to the remote tip read right after a successful fetch. Returns
4847    /// the reason when the run must stop; `None` when there was nothing to do
4848    /// (no remote copy, the same tip, or a remote copy this branch already
4849    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4850    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4851        // Task competitions publish only after the PR text gate, to their
4852        // public head; never resurrect a candidate branch on the remote.
4853        if self.state.task_pr_branch().is_some() {
4854            return None;
4855        }
4856        let repo = self.state.repo.clone();
4857        let remote = self.state.config.merge.remote.clone();
4858        let fetched = git::fetch(&repo, &remote, branch).await;
4859        if !matches!(&fetched, Ok(o) if o.ok()) {
4860            return None;
4861        }
4862        let branch_tracking = format!("{remote}/{branch}");
4863        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4864        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4865            return None;
4866        }
4867        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4868            .await
4869            .is_ok_and(|missing| missing.is_empty())
4870        {
4871            return Some(format!(
4872                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4873                 them",
4874                short(&theirs)
4875            ));
4876        }
4877        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4878            Ok(o) if o.ok() => {
4879                self.state
4880                    .event("land", format!("pushed rebased {branch} to {remote}"));
4881                None
4882            }
4883            Ok(o) => Some(format!(
4884                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4885                 someone may have pushed): {}",
4886                short(&theirs),
4887                o.stderr.chars().take(600).collect::<String>()
4888            )),
4889            Err(e) => Some(format!(
4890                "{branch} is rebased locally but could not be pushed: {e:#}"
4891            )),
4892        }
4893    }
4894
4895    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4896    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4897    /// whether it did.
4898    ///
4899    /// Checked only when the base is ahead of the branch. A failing check is
4900    /// "not proven" - the ordinary rebase path then decides - never a reason to
4901    /// stop the run.
4902    ///
4903    /// The remote copy of the branch is held to the same standard as the local
4904    /// one: if it carries a tip this worktree does not, that tip must itself be
4905    /// proven in the base, or nothing is settled (a pull request would
4906    /// otherwise be closed over commits nobody checked). The pull request is
4907    /// closed *before* the terminal status is saved; if that fails for a
4908    /// reason other than a refusal (no network, a `gh` error) the run is left
4909    /// `Blocked` with the reason as its conflict, which a resume retries -
4910    /// the same recovery a phantom conflict gets.
4911    async fn settle_already_in(
4912        &mut self,
4913        branch: &str,
4914        tip: &str,
4915        head: &str,
4916        attempts: usize,
4917        behind: usize,
4918    ) -> Result<bool> {
4919        let repo = self.state.repo.clone();
4920        let remote = self.state.config.merge.remote.clone();
4921        let start = self.state.base_commit.clone();
4922        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4923            Ok(Some(e)) => e,
4924            Ok(None) => return Ok(false),
4925            Err(e) => {
4926                tracing::warn!("already-in-base check for {branch}: {e:#}");
4927                return Ok(false);
4928            }
4929        };
4930        let mut verified = vec![head.to_owned()];
4931        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4932        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4933        if matches!(&fetched, Ok(o) if o.ok())
4934            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4935            && theirs != head
4936        {
4937            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4938                Ok(Some(_)) => verified.push(theirs),
4939                _ => return Ok(false),
4940            }
4941        }
4942        let base_branch = self.state.base_branch.clone();
4943        let message = format!(
4944            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4945             land",
4946            evidence.names(),
4947            evidence.proof.as_str()
4948        );
4949        let closed =
4950            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4951                .await;
4952        match closed {
4953            Ok(Ok(url)) => self
4954                .state
4955                .event("land", format!("closed {url}: superseded on {base_branch}")),
4956            Ok(Err(why)) => self
4957                .state
4958                .event("land", format!("did not close a pull request: {why}")),
4959            Err(e) => {
4960                let why = format!(
4961                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
4962                     not be closed ({e:#}); resume to retry"
4963                );
4964                self.state.status = RunStatus::Blocked;
4965                self.state.base_sync = Some(BaseSync {
4966                    tip: tip.to_owned(),
4967                    behind,
4968                    attempts,
4969                    conflict: Some(why.clone()),
4970                    already_in: None,
4971                });
4972                self.state.event("land", why);
4973                self.state.save()?;
4974                return Ok(true);
4975            }
4976        }
4977        self.state.status = RunStatus::AlreadyInBase;
4978        self.state.base_sync = Some(BaseSync {
4979            tip: tip.to_owned(),
4980            behind,
4981            attempts,
4982            conflict: None,
4983            already_in: Some(evidence),
4984        });
4985        self.state.event("land", message);
4986        self.state.save()?;
4987        self.settle_questions();
4988        Ok(true)
4989    }
4990
4991    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
4992    /// last landed the winner on, once it has run, else the commit the run
4993    /// branched from.
4994    ///
4995    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
4996    /// and `vote` all happen before there is a winner to rebase, so they
4997    /// compare every candidate against the branch point on purpose, and a
4998    /// base that moves after they are already done cannot change an answer
4999    /// they already gave.
5000    fn landing_base(&self) -> String {
5001        self.state
5002            .base_sync
5003            .as_ref()
5004            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5005    }
5006
5007    // ------------------------------------------------------- operator fix
5008
5009    /// Route specific, already-recorded review findings to a fixer for a
5010    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5011    /// entry point.
5012    ///
5013    /// Distinct from `review_loop`'s own fix step in three ways: it never
5014    /// runs a reviewer wave, it never spends review-round budget, and what
5015    /// happened is recorded as an [`OperatorFixRequest`] appended to
5016    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5017    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5018    /// and vote must never be rewritten to look like a manufactured blocking
5019    /// verdict.
5020    ///
5021    /// Only meaningful once review has actually concluded: `Ready` (handed
5022    /// off with findings still open, or simply concluded clean while minor
5023    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5024    /// gate failed). Everything else is refused: a run still in progress
5025    /// should simply be resumed, and a `Merged` run's branch has already
5026    /// landed — reopening *this* run's own record cannot change that, so the
5027    /// answer there is a fresh `magi review <branch>`.
5028    ///
5029    /// A real commit here re-verifies through a fresh, ordinary review-only
5030    /// run on the same branch ([`Self::review`]) rather than reopening this
5031    /// run's own `review_loop`: once any round in this run's history went
5032    /// clean, `review_conclusion` treats that as permanent by design (the
5033    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5034    /// way to force one more genuine reviewer wave out of *this* run without
5035    /// either rewriting history or weakening that guarantee for every other
5036    /// caller. A review-only run costs nothing extra — no implementation, no
5037    /// judging, no vote — and exercises the exact same review → verify →
5038    /// gate → (human) merge path, unmodified.
5039    pub async fn fix_selected(
5040        &mut self,
5041        ids: &[String],
5042        reason: &str,
5043        allow_stale: bool,
5044    ) -> Result<()> {
5045        let reason = reason.trim();
5046        if reason.is_empty() {
5047            bail!("a fix request needs a reason — that is the operator's own record of why");
5048        }
5049        if ids.is_empty() {
5050            bail!("no finding id given");
5051        }
5052        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5053            bail!(
5054                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5055                 has already concluded — can be given a targeted fix. A run still \
5056                 in progress should simply be resumed; a `merged` run's branch has \
5057                 already landed, so its answer is a fresh `magi review <branch>`, \
5058                 not reopening this run's own record",
5059                self.state.id,
5060                self.state.status.as_str()
5061            );
5062        }
5063        let Some(winner) = self.state.winner().cloned() else {
5064            bail!("run {} has no winning candidate to fix", self.state.id);
5065        };
5066        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5067            bail!(
5068                "branch `{}` no longer exists; this run cannot be extended",
5069                winner.branch
5070            );
5071        }
5072        let home = crate::run::home();
5073        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5074            bail!(
5075                "run {} is currently being worked on by another magi process",
5076                self.state.id
5077            );
5078        }
5079        // Held for the rest of this call, including the follow-up review
5080        // below: two `magi fix` invocations against the same run must not
5081        // both reach the worktree manipulation further down, which would
5082        // otherwise race to remove and recreate the same directory — see
5083        // [`FixClaim`]'s own doc.
5084        let _claim = FixClaim::acquire(&self.state.dir())?;
5085
5086        // Resolve every id before spending anything — an unknown id refuses
5087        // the whole request rather than silently dropping it — and dedup
5088        // while keeping the operator's own order.
5089        let mut seen = BTreeSet::new();
5090        let mut findings = Vec::new();
5091        let mut missing = Vec::new();
5092        for id in ids {
5093            if !seen.insert(id.clone()) {
5094                continue;
5095            }
5096            match self.state.finding(id) {
5097                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5098                    id: f.id.clone(),
5099                    severity: f.severity,
5100                    reviewer_vote: rec.vote,
5101                    round: round.round,
5102                    round_head: round.head.clone(),
5103                    reviewer: rec.reviewer,
5104                    agent: rec.agent.clone(),
5105                    file: f.file.clone(),
5106                    line: f.line,
5107                    title: f.title.clone(),
5108                    detail: f.detail.clone(),
5109                    outcome: OperatorFixOutcome::Pending,
5110                }),
5111                None => missing.push(id.clone()),
5112            }
5113        }
5114        if !missing.is_empty() {
5115            bail!(
5116                "unknown finding id(s): {}; nothing was changed",
5117                missing.join(", ")
5118            );
5119        }
5120
5121        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5122        let stale_details: Vec<(String, String)> = findings
5123            .iter()
5124            .filter(|f| f.round_head != head_at_request)
5125            .map(|f| (f.id.clone(), f.round_head.clone()))
5126            .collect();
5127        let stale = !stale_details.is_empty();
5128        if stale && !allow_stale {
5129            bail!(
5130                "the branch has moved since some finding(s) were raised — {} — now \
5131                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5132                stale_details
5133                    .iter()
5134                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5135                    .collect::<Vec<_>>()
5136                    .join(", "),
5137                short(&head_at_request)
5138            );
5139        }
5140
5141        let request = OperatorFixRequest {
5142            requested_at: Timestamp::now(),
5143            reason: reason.to_owned(),
5144            findings,
5145            head_at_request: head_at_request.clone(),
5146            allow_stale,
5147            stale,
5148            fix: None,
5149            result_head: None,
5150            follow_up_review_run: None,
5151        };
5152        self.state.event(
5153            "fix",
5154            format!(
5155                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5156                request.findings.len(),
5157                request
5158                    .findings
5159                    .iter()
5160                    .map(|f| f.id.as_str())
5161                    .collect::<Vec<_>>()
5162                    .join(", "),
5163            ),
5164        );
5165        // Recorded now, before any worktree work or the fixer call itself —
5166        // and re-saved at each checkpoint below: a crash at any point after
5167        // this (mid fixer call, mid follow-up review) must not lose the fact
5168        // that this was requested, for which findings, and why. Everything
5169        // past this point reads and writes through `request_index` rather
5170        // than a local variable, since `request` itself is moved here.
5171        self.state.operator_fixes.push(request);
5172        self.state.save()?;
5173        let request_index = self.state.operator_fixes.len() - 1;
5174
5175        // A fresh, dedicated worktree for this one call, never the winner's
5176        // own worktree in place: that one may already be gone (folded away),
5177        // and reusing it in place would leave the branch checked out there
5178        // when the follow-up review below tries to check it out again. Freed
5179        // immediately after, either way — but only once confirmed clean:
5180        // `worktree_remove` is a `git worktree remove --force`, which would
5181        // otherwise discard uncommitted work left there by the operator or
5182        // another process before this had a chance to even look at it.
5183        let repo = self.state.repo.clone();
5184        let coordination = self.coordinate(&repo).await?;
5185        if winner.worktree.exists() {
5186            // Lockfiles a rescue commit withheld stay untracked on purpose and
5187            // are already recorded; they are not the operator's work to protect.
5188            let dirty = git::git(
5189                &winner.worktree,
5190                &["status", "--porcelain", "--untracked-files=all"],
5191            )
5192            .await?;
5193            let only_withheld = dirty.lines().all(|l| {
5194                l.strip_prefix("?? ")
5195                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5196            });
5197            if !only_withheld {
5198                bail!(
5199                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5200                     discard them first",
5201                    winner.worktree.display()
5202                );
5203            }
5204            git::worktree_remove(&self.state.repo, &winner.worktree)
5205                .await
5206                .ok();
5207        }
5208        let fix_worktree = self.state.worktree_root().join("operator-fix");
5209        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5210        git::git(
5211            &self.state.repo,
5212            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5213        )
5214        .await
5215        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5216        if !git::is_clean(&fix_worktree).await? {
5217            git::worktree_remove(&self.state.repo, &fix_worktree)
5218                .await
5219                .ok();
5220            bail!(
5221                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5222                winner.branch
5223            );
5224        }
5225        let repo_for_setup = self.state.repo.clone();
5226        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5227            Ok(()) => {
5228                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5229            }
5230            Err(e) => Err(e),
5231        };
5232        if let Err(e) = setup {
5233            git::worktree_remove(&self.state.repo, &fix_worktree)
5234                .await
5235                .ok();
5236            return Err(e);
5237        }
5238
5239        let run_id = self.state.id.clone();
5240        let prompts = self.state.config.prompts.clone();
5241        let language = self.state.config.graph.language.clone();
5242        let sessions = self.state.config.graph.sessions;
5243        let artifacts = agent::artifacts_dir(&self.state.dir());
5244        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5245            .findings
5246            .iter()
5247            .map(|f| Finding {
5248                id: f.id.clone(),
5249                severity: f.severity,
5250                file: f.file.clone(),
5251                line: f.line,
5252                title: f.title.clone(),
5253                detail: f.detail.clone(),
5254            })
5255            .collect();
5256        let fix_prompt = prompt::operator_fix_in(
5257            &self.state.instruction,
5258            &finding_list,
5259            reason,
5260            &stale_details,
5261            &head_at_request,
5262            &language,
5263            &self.state.config.graph.github_language,
5264        );
5265        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5266        drop(coordination);
5267        let (job, seat, out) = self
5268            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5269                prompt: fix_prompt.clone(),
5270                spec,
5271                seat,
5272                cwd: fix_worktree.clone(),
5273                timeout,
5274                allow_write: true,
5275                sessions,
5276                artifacts: artifacts.clone(),
5277                stem: "operator-fix".to_owned(),
5278                handover: None,
5279            })
5280            .await;
5281        let agent_id = seat.agent.clone();
5282
5283        let mut fix = FixRecord {
5284            agent: agent_id,
5285            addressed: Vec::new(),
5286            rejected: Vec::new(),
5287            notes: String::new(),
5288            committed: false,
5289            failed: None,
5290            duration_ms: 0,
5291            continuation: None,
5292        };
5293        let mut final_seat = seat.clone();
5294        match out {
5295            AgentOutcome::Ok(o) => {
5296                fix.duration_ms = o.duration_ms;
5297                let parsed = verdict::extract_json::<FixReport>(&o.text);
5298                let incomplete_reason = match &parsed {
5299                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5300                        "the reply parsed, but it reported a command whose own CLI \
5301                         never confirmed an exit status"
5302                            .to_owned(),
5303                    ),
5304                    Ok(_) => None,
5305                    Err(e) => Some(e.to_string()),
5306                };
5307                match incomplete_reason {
5308                    None => {
5309                        let report = parsed.expect("checked Ok above");
5310                        fix.addressed = report.addressed;
5311                        fix.rejected = report.rejected;
5312                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5313                    }
5314                    Some(reason) => {
5315                        let (resumed_seat, resolved, failure, cont) = self
5316                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5317                            .await;
5318                        fix.duration_ms += cont.cumulative_wait_ms;
5319                        fix.continuation = Some(cont);
5320                        final_seat = resumed_seat;
5321                        match resolved {
5322                            Some(report) => {
5323                                fix.addressed = report.addressed;
5324                                fix.rejected = report.rejected;
5325                                fix.notes =
5326                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5327                            }
5328                            None => fix.failed = failure,
5329                        }
5330                    }
5331                }
5332            }
5333            AgentOutcome::Dropped(o) => {
5334                fix.duration_ms = o.duration_ms;
5335                let why = o
5336                    .dropped
5337                    .as_ref()
5338                    .map(|d| d.why.as_str())
5339                    .unwrap_or("the CLI ended the stream without delivering its answer");
5340                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5341            }
5342            AgentOutcome::Quota(o) => {
5343                self.state.quota.push(QuotaLoss {
5344                    seat: final_seat.key.clone(),
5345                    node: "fix".to_owned(),
5346                    at: Timestamp::now(),
5347                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5348                });
5349                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5350            }
5351            AgentOutcome::Failed(e) => fix.failed = Some(e),
5352            AgentOutcome::ResourceBlocked(e) => {
5353                fix.failed = Some(format!("the fixer was not started: {e}"));
5354            }
5355        }
5356        if fix.continuation.is_none() {
5357            fix.continuation = Some(ContinuationRecord::not_needed());
5358        }
5359        self.state.seats.insert(final_seat.key.clone(), final_seat);
5360
5361        let rescue_message = format!(
5362            "magi: operator-selected fix ({}) (uncommitted work)",
5363            self.state.operator_fixes[request_index]
5364                .findings
5365                .iter()
5366                .map(|f| f.id.as_str())
5367                .collect::<Vec<_>>()
5368                .join(", ")
5369        );
5370        let repo = self.state.repo.clone();
5371        let coordination = self.coordinate(&repo).await?;
5372        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5373            self.state.note_withheld("fix", &r.withheld);
5374        }
5375        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5376        fix.committed = after != head_at_request;
5377        git::worktree_remove(&self.state.repo, &fix_worktree)
5378            .await
5379            .ok();
5380        drop(coordination);
5381
5382        self.state.event(
5383            "fix",
5384            match &fix.failed {
5385                Some(reason) => format!(
5386                    "operator fix: adoption report was lost ({reason}); {}",
5387                    if fix.committed {
5388                        "committed"
5389                    } else {
5390                        "NO new commit"
5391                    }
5392                ),
5393                None => format!(
5394                    "operator fix: {} addressed, {} rejected, {}",
5395                    fix.addressed.len(),
5396                    fix.rejected.len(),
5397                    if fix.committed {
5398                        "committed"
5399                    } else {
5400                        "NO new commit"
5401                    }
5402                ),
5403            },
5404        );
5405
5406        // Every selected finding gets an outcome — never left `Pending` once
5407        // the fixer's own turn is over. A report that never came back at all
5408        // marks every one of them `Unreported`, not silently "not addressed":
5409        // quota, a dropped stream, or an exhausted continuation are gaps in
5410        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5411        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5412        for f in &mut self.state.operator_fixes[request_index].findings {
5413            f.outcome = if fix.failed.is_some() {
5414                OperatorFixOutcome::Unreported
5415            } else if fix.addressed.contains(&f.id) {
5416                OperatorFixOutcome::Addressed
5417            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5418                OperatorFixOutcome::Rejected { why: r.why.clone() }
5419            } else {
5420                OperatorFixOutcome::Unreported
5421            };
5422        }
5423
5424        let committed = fix.committed;
5425        if committed {
5426            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5427        }
5428        self.state.operator_fixes[request_index].fix = Some(fix);
5429        // Saved again now that the fixer's own outcome is final, on top of
5430        // the save right after the request was first pushed above.
5431        self.state.save()?;
5432
5433        if committed {
5434            self.state.event(
5435                "fix",
5436                format!(
5437                    "operator fix committed {}; opening a follow-up review-only run",
5438                    short(&after)
5439                ),
5440            );
5441            // The operator asked for the fix, and the follow-up serves whatever
5442            // task the run it follows served.
5443            let origin =
5444                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5445            match Self::review(
5446                &self.state.repo,
5447                &winner.branch,
5448                self.state.config.clone(),
5449                origin,
5450            )
5451            .await
5452            {
5453                Ok(mut follow_up) => {
5454                    follow_up.state.event(
5455                        "start",
5456                        format!(
5457                            "requested by an operator fix on run {} for finding(s) {}",
5458                            self.state.id,
5459                            self.state.operator_fixes[request_index]
5460                                .findings
5461                                .iter()
5462                                .map(|f| f.id.as_str())
5463                                .collect::<Vec<_>>()
5464                                .join(", "),
5465                        ),
5466                    );
5467                    follow_up.state.save()?;
5468                    let follow_up_id = follow_up.state.id.clone();
5469                    // The follow-up is a run like any other: it belongs to the
5470                    // task of the run it follows, or to one filed for it.
5471                    let adopted = match crate::direct::adopt(
5472                        &follow_up.state,
5473                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5474                    ) {
5475                        Ok(a) => a,
5476                        Err(e) => {
5477                            // An ownerless run must not spend agent calls; it
5478                            // stays saved, and `magi run --resume` adopts it.
5479                            self.state.event(
5480                                "fix",
5481                                format!(
5482                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5483                                ),
5484                            );
5485                            self.state.save()?;
5486                            return Ok(());
5487                        }
5488                    };
5489                    let executed = follow_up.execute().await;
5490                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5491                    if let Some(a) = adopted {
5492                        a.finish(&follow_up.state, executed);
5493                    }
5494                    if let Some(e) = failure {
5495                        self.state.event(
5496                            "fix",
5497                            format!(
5498                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5499                            ),
5500                        );
5501                    }
5502                    self.state.operator_fixes[request_index].follow_up_review_run =
5503                        Some(follow_up_id);
5504                }
5505                Err(e) => {
5506                    self.state.event(
5507                        "fix",
5508                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5509                    );
5510                }
5511            }
5512            self.state.save()?;
5513        }
5514
5515        Ok(())
5516    }
5517
5518    // --------------------------------------------------------------- review
5519
5520    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5521    /// names, in order, each at most once, moving on only when the call
5522    /// advances (an error, a quota hit or nothing usable - the same decision
5523    /// point `agent::chain_advances` is for the other chained roles).
5524    ///
5525    /// Each agent gets its own seat from `Runner::seat`: the same agent
5526    /// continues its conversation (the winner's own implementer seat when it
5527    /// is the winner's author, now that the competition is over), another
5528    /// takes a fresh one so the full prompt is sent again. A handover is
5529    /// recorded under `node`, which is what makes the fallback stick for the
5530    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5531    /// is returned, so an exhausted chain reads exactly like a single failed
5532    /// fixer: one quota loss, the same wording, the same refund. An earlier
5533    /// attempt's edits are left in the tree and judged, with the final
5534    /// attempt's, by what git says afterwards.
5535    ///
5536    /// Returns the job that produced the outcome, for `continue_fix_report`.
5537    async fn ask_fixer(
5538        &mut self,
5539        winner: &Candidate,
5540        node: &'static str,
5541        round: Option<usize>,
5542        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5543    ) -> (SeatJob, SeatState, AgentOutcome) {
5544        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5545        self.ask_fixer_among(attempts, node, round, None, build)
5546            .await
5547    }
5548
5549    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5550    /// worktree and a commit: before the next agent is asked after one that
5551    /// did not satisfy the call, the worktree is put back on that commit, so
5552    /// a failed agent's partial edits are not carried into the next one.
5553    async fn ask_fixer_among(
5554        &mut self,
5555        attempts: Vec<(AgentSpec, String)>,
5556        node: &'static str,
5557        round: Option<usize>,
5558        restore: Option<(&Path, &str)>,
5559        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5560    ) -> (SeatJob, SeatState, AgentOutcome) {
5561        let run_id = self.state.id.clone();
5562        let prompts = self.state.config.prompts.clone();
5563        let cache = self.state.config.cache_dir();
5564        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5565        let mut last = None;
5566        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5567            let seat = self.seat(&key, &spec.id);
5568            let mut job = build(spec.clone(), seat);
5569            if i > 0 {
5570                job.stem = format!("{}-{}", job.stem, spec.id);
5571            }
5572            let ctx = WaveCtx {
5573                carry_seats: false,
5574                pauses: Some((&self.pause, &self.interrupt)),
5575                run: &run_id,
5576                node,
5577                prompts: &prompts,
5578                cache: cache.as_deref(),
5579                round,
5580            };
5581            let (seat, out) =
5582                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5583            // A resource block never advances the chain: the chosen agent
5584            // was not asked, so it must be the one asked again on resume.
5585            let advances = match &out {
5586                AgentOutcome::Ok(o) => agent::output_advances(o),
5587                AgentOutcome::ResourceBlocked(_) => false,
5588                _ => true,
5589            };
5590            if let Some(next) = ids.get(i + 1)
5591                && advances
5592            {
5593                self.state.seats.insert(seat.key.clone(), seat.clone());
5594                let class =
5595                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5596                let reason = match &out {
5597                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5598                    other => fail_reason(other),
5599                };
5600                // A failed agent's edits must not reach the next one, and a
5601                // restore that did not work ends the chain: the call fails
5602                // and the caller's own (confirmed) restore decides.
5603                if let Some((wt, rev)) = restore
5604                    && let Err(e) = self.graft_restore(wt, rev).await
5605                {
5606                    let why =
5607                        format!("could not restore the worktree before the next fixer: {e:#}");
5608                    tracing::warn!("{why}");
5609                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5610                        AgentOutcome::ResourceBlocked(why)
5611                    } else {
5612                        AgentOutcome::Failed(why)
5613                    };
5614                    last = Some((job, seat, out));
5615                    break;
5616                }
5617                record_handover(
5618                    &mut self.state,
5619                    node,
5620                    &seat.key,
5621                    &spec.id,
5622                    next,
5623                    &class,
5624                    &reason,
5625                );
5626                continue;
5627            }
5628            last = Some((job, seat, out));
5629            break;
5630        }
5631        last.expect("the fixer chain always has an entry")
5632    }
5633
5634    /// The one repair a verification-only failure earns (see
5635    /// [`crate::recovery`]), before the first review round of a run that took
5636    /// a branch over.
5637    ///
5638    /// Saved as started *before* the fixer is called, so a resume never
5639    /// repairs twice; a start found on re-entry is rolled back to the commit
5640    /// it began from. A failed or unusable fixer leaves the tree as it was and
5641    /// the review goes on over it, so the first round's own verification
5642    /// reports the failure again. The failure is never carried into a round as
5643    /// a result: verification is re-run on whatever commit this leaves.
5644    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5645        use crate::recovery::{Mode, Repair};
5646        if !self.state.reviews.is_empty() {
5647            return Ok(());
5648        }
5649        let Some(mut rec) = self.state.recovery.clone() else {
5650            return Ok(());
5651        };
5652        if rec.mode != Mode::VerifyRepair {
5653            return Ok(());
5654        }
5655        let wt = winner.worktree.clone();
5656        let mut head = git::rev_parse(&wt, "HEAD").await?;
5657        match rec.repair.clone() {
5658            Repair::Done { .. } => return Ok(()),
5659            Repair::WaitingForRestore { before } => {
5660                self.graft_restore(&wt, &before).await?;
5661                head = before;
5662                rec.repair = Repair::NotStarted;
5663                self.state.recovery = Some(rec.clone());
5664                self.state.save()?;
5665            }
5666            Repair::Started { before } => {
5667                // Interrupted: the unfinished edits are not trusted.
5668                // An unconfirmed rollback blocks the run with `Started` kept,
5669                // so a resume retries it; the tree is never reviewed as is.
5670                if let Err(e) = self.graft_restore(&wt, &before).await {
5671                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5672                        return Err(e);
5673                    }
5674                    return self
5675                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5676                        .await;
5677                }
5678                rec.repair = Repair::Done {
5679                    before: before.clone(),
5680                    after: before,
5681                    committed: false,
5682                    failed: Some("interrupted; rolled back".to_owned()),
5683                };
5684                self.state
5685                    .event("recovery", "the repair was interrupted and rolled back");
5686                self.state.recovery = Some(rec);
5687                self.state.save()?;
5688                return Ok(());
5689            }
5690            Repair::NotStarted => {}
5691        }
5692        if crate::recovery::recheck(&mut rec, &head) {
5693            self.state.event("recovery", rec.summary());
5694            self.state.recovery = Some(rec);
5695            self.state.save()?;
5696            return Ok(());
5697        }
5698        rec.repair = Repair::Started {
5699            before: head.clone(),
5700        };
5701        self.state.recovery = Some(rec.clone());
5702        self.state.save()?;
5703
5704        let language = self.state.config.graph.language.clone();
5705        let sessions = self.state.config.graph.sessions;
5706        let artifacts = agent::artifacts_dir(&self.state.dir());
5707        let fix_prompt = prompt::recovery_fix(
5708            &self.state.instruction,
5709            &rec,
5710            self.state.config.graph.review_rounds,
5711            &language,
5712        );
5713        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5714        self.state.event(
5715            "recovery",
5716            format!(
5717                "repair: the fixer starts from the failed verification on {}",
5718                short(&head)
5719            ),
5720        );
5721        let (_job, seat, out) = self
5722            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5723                prompt: fix_prompt.clone(),
5724                spec,
5725                seat,
5726                cwd: wt.clone(),
5727                timeout,
5728                allow_write: true,
5729                sessions,
5730                artifacts: artifacts.clone(),
5731                stem: "fix-recovery".to_owned(),
5732                handover: None,
5733            })
5734            .await;
5735        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5736            rec.repair = Repair::WaitingForRestore { before: head };
5737            self.state.recovery = Some(rec);
5738            return self.stop_for_resource();
5739        }
5740        let failed = match &out {
5741            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5742            AgentOutcome::Quota(o) => {
5743                self.state.quota.push(QuotaLoss {
5744                    seat: seat.key.clone(),
5745                    node: "fix".to_owned(),
5746                    at: Timestamp::now(),
5747                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5748                });
5749                Some("rate limited (quota); the fixer could not run".to_owned())
5750            }
5751            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5752            AgentOutcome::Failed(e) => Some(e.clone()),
5753        };
5754        self.state.seats.insert(seat.key.clone(), seat);
5755        let coordination = self.coordinate(&wt).await?;
5756        let failed = match failed {
5757            // The agent's partial edits are not kept.
5758            Some(why) => {
5759                if let Err(e) = git::restore_to(&wt, &head).await {
5760                    return self
5761                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5762                        .await;
5763                }
5764                Some(why)
5765            }
5766            None => {
5767                if let Ok(r) = git::rescue_commit(
5768                    &wt,
5769                    "magi: recovery repair of the failed verification (uncommitted work)",
5770                )
5771                .await
5772                {
5773                    self.state.note_withheld("fix", &r.withheld);
5774                }
5775                None
5776            }
5777        };
5778        let after = git::rev_parse(&wt, "HEAD").await?;
5779        drop(coordination);
5780        self.state.event(
5781            "recovery",
5782            match &failed {
5783                Some(why) => format!("repair: nothing kept ({why})"),
5784                None if after != head => {
5785                    format!("repair: committed {}", short(&after))
5786                }
5787                None => "repair: the fixer made no commit".to_owned(),
5788            },
5789        );
5790        rec.repair = Repair::Done {
5791            before: head.clone(),
5792            committed: after != head,
5793            after,
5794            failed,
5795        };
5796        self.state.recovery = Some(rec);
5797        self.state.save()?;
5798        Ok(())
5799    }
5800
5801    /// A repair whose rollback could not be confirmed: the run is blocked and
5802    /// resumable, and the unconfirmed state is never reviewed.
5803    async fn stop_recovery(
5804        &mut self,
5805        mut rec: crate::recovery::RecoveryRecord,
5806        head: &str,
5807        why: String,
5808    ) -> Result<()> {
5809        // `Started` stays recorded: a resume retries the restore.
5810        rec.repair = crate::recovery::Repair::Started {
5811            before: head.to_owned(),
5812        };
5813        self.state.recovery = Some(rec);
5814        self.state
5815            .event("recovery", format!("repair could not be undone: {why}"));
5816        self.state.status = RunStatus::Blocked;
5817        self.state.save()?;
5818        bail!("recovery repair could not be rolled back: {why}")
5819    }
5820
5821    async fn review_loop(&mut self) -> Result<()> {
5822        // A base that would not rebase is a person's decision, not a review
5823        // round: nothing here would change the answer, and reviewers and a
5824        // fixer would be spending real budget on a tree that cannot land
5825        // regardless of what they find.
5826        if self
5827            .state
5828            .base_sync
5829            .as_ref()
5830            .is_some_and(|s| s.conflict.is_some())
5831        {
5832            return Ok(());
5833        }
5834        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5835        // agent files with `magi task add` name the run that paid for it. The
5836        // prompt overlay is cloned alongside it because the waves borrow it
5837        // while `self` is mutably borrowed by the node's own bookkeeping.
5838        let run_id = self.state.id.clone();
5839        let prompts = self.state.config.prompts.clone();
5840        let Some(winner) = self.state.winner().cloned() else {
5841            return Ok(());
5842        };
5843        let max_rounds = self.state.config.graph.review_rounds;
5844        // A clean round, an exhausted round budget, or a stalled tree (see
5845        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5846        // they are recorded — recomputed here, not read off `status`, so a
5847        // reentry into a run that already stopped restates the identical
5848        // verdict instead of silently handing back whatever an earlier node
5849        // in this same walk clobbered `status` to (a solo-candidate
5850        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5851        // below runs an empty range once the budget is spent, and would
5852        // otherwise fall through without touching `status` at all.
5853        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5854            // A reentry after a crash between the last round's save and
5855            // `stop_reviewing` reaches the hand-off here, not there.
5856            if status == RunStatus::Gating {
5857                self.record_contested_handoff();
5858            }
5859            self.state.status = status;
5860            self.state.save()?;
5861            return Ok(());
5862        }
5863        self.state.status = RunStatus::Reviewing;
5864        // A last recorded round whose own verification never resolved
5865        // (`ResourceBlocked` — the shared build cache, not the patch) is
5866        // never a concluded round, whatever the round budget says: starting
5867        // a fresh round on top of it would spend a whole new reviewer wave
5868        // re-reading an unchanged patch instead of just retrying the one
5869        // check that actually needs it, and once the budget is spent the
5870        // loop below has nothing left to do at all (its range is empty).
5871        // Retry that check directly instead, exactly the same retry
5872        // `stop_reviewing` already does for its own catch-up case.
5873        if self
5874            .state
5875            .reviews
5876            .last()
5877            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5878        {
5879            let shell = self.state.config.shell();
5880            return self
5881                .stop_reviewing(
5882                    "the last round's own verification never resolved",
5883                    &shell,
5884                    &winner.worktree,
5885                )
5886                .await;
5887        }
5888
5889        self.recovery_repair(&winner).await?;
5890        if self.resource_halt()? {
5891            return Ok(());
5892        }
5893
5894        let repo = self.state.repo.clone();
5895        let root = self.state.worktree_root();
5896        let language = self.state.config.graph.language.clone();
5897        let sessions = self.state.config.graph.sessions;
5898        let artifacts = self.merge_repair_artifacts();
5899        let base = self.landing_base();
5900        let base_short = short(&base);
5901        let reviewers = self.roles.reviewers.clone();
5902        let shell = self.state.config.shell();
5903
5904        for round in (self.state.reviews.len() + 1)..=max_rounds {
5905            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5906            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5907            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5908            // The prior round's own record, already persisted — never a
5909            // hand-carried variable of just its failing output: that is
5910            // exactly what let a round's e2e result drift out of sync with
5911            // which commit it was actually about (see `SCHEMA`'s doc for
5912            // schema 8). Judged against `head`, the commit reviewers are
5913            // about to look at now, so the summary always reads as "an
5914            // earlier head" here — this round's own patch has not been
5915            // checked yet.
5916            let prev_verification = self
5917                .state
5918                .reviews
5919                .last()
5920                .and_then(|r| r.verification_summary(&head));
5921            // Only the first round of a run that took a branch over: later
5922            // rounds have this run's own history.
5923            let prior_section = match (&self.state.recovery, round) {
5924                (Some(rec), 1) if self.state.reviews.is_empty() => {
5925                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5926                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5927                            Some(crate::recovery::head_of(
5928                                &git::diff(&winner.worktree, reviewed, "HEAD")
5929                                    .await
5930                                    .unwrap_or_default(),
5931                                crate::recovery::SINCE_DIFF_MAX,
5932                            ))
5933                        }
5934                        _ => None,
5935                    };
5936                    Some(prompt::recovery_review(rec, since.as_deref()))
5937                }
5938                _ => None,
5939            };
5940
5941            // Each reviewer gets its own detached checkout of exactly this
5942            // commit: nobody can perturb the winner's tree, and the fixer can
5943            // keep working without racing a reviewer.
5944            let mut jobs = Vec::new();
5945            let coordination = self.coordinate(&repo).await?;
5946            for (r, spec) in reviewers.iter().cloned().enumerate() {
5947                let wt = root.join(format!("review-{}", r + 1));
5948                // `clean -fdx` in a reset wipes setup's products, so every
5949                // round sets the seat up again.
5950                if wt.exists() {
5951                    git::reset_detached(&wt, &head).await?;
5952                } else {
5953                    git::worktree_add_detached(&repo, &wt, &head).await?;
5954                }
5955                self.setup_seat_worktree(&repo, &wt, false).await?;
5956                let seat_key = format!("review-{}", r + 1);
5957                // The seat starts the round on whoever answered it last, not
5958                // on the agent the spec names, so a failure is not re-paid.
5959                let spec = pick_start_spec(
5960                    &self.roles.reviewer_roster,
5961                    spec,
5962                    self.state.seat_history.get(&seat_key),
5963                );
5964                let seat = self.seat(&seat_key, &spec.id);
5965                jobs.push(SeatJob {
5966                    prompt: prompt::review(&prompt::ReviewCtx {
5967                        instruction: &self.state.instruction,
5968                        branch: &winner.branch,
5969                        base_short: &base_short,
5970                        stat: &stat,
5971                        patch: &patch,
5972                        verification: prev_verification.as_ref(),
5973                        prior: prior_section.as_deref(),
5974                        reviewers: reviewers.len(),
5975                        round,
5976                        rounds: max_rounds,
5977                        // A review-only run has no rankings, so nothing
5978                        // competed for this patch and the reviewer is told so.
5979                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5980                        lens: Lens::for_seat(r),
5981                        language: &language,
5982                        github_language: &self.state.config.graph.github_language,
5983                    }),
5984                    spec,
5985                    seat,
5986                    cwd: wt,
5987                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5988                    allow_write: false,
5989                    sessions,
5990                    artifacts: artifacts.clone(),
5991                    stem: format!("review-{round}-{}", r + 1),
5992                    handover: None,
5993                });
5994            }
5995
5996            drop(coordination);
5997            self.state.event(
5998                "review",
5999                format!(
6000                    "round {round}: {} reviewers on {}",
6001                    jobs.len(),
6002                    short(&head)
6003                ),
6004            );
6005            let mut quota_losses = Vec::new();
6006            let review_retries = self.state.config.graph.retries;
6007            let review_cache = self.state.config.cache_dir();
6008            let ctx = WaveCtx {
6009                carry_seats: true,
6010                pauses: Some((&self.pause, &self.interrupt)),
6011                run: &run_id,
6012                node: "review",
6013                prompts: &prompts,
6014                cache: review_cache.as_deref(),
6015                round: Some(round),
6016            };
6017            let results = ask_json_wave::<Review>(
6018                jobs,
6019                Arc::clone(&self.sem),
6020                review_retries,
6021                &self.roles.reviewer_roster,
6022                &ctx,
6023                &mut quota_losses,
6024                &mut self.state,
6025                &|_: &Review| Ok(()),
6026            )
6027            .await;
6028            if self.state.resource_block.is_some() {
6029                return self.stop_for_resource();
6030            }
6031            // Counted before the move below: how many of *this* round's
6032            // reviewer seats were lost to their own rate limit, as opposed to
6033            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6034            let round_quota_missing = quota_losses.len();
6035            self.state.quota.extend(quota_losses);
6036
6037            let mut records = Vec::new();
6038            let mut all_findings = Vec::new();
6039            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6040                let agent_id = seat.agent.clone();
6041                self.state.seats.insert(seat.key.clone(), seat);
6042                let mut record = ReviewRecord {
6043                    reviewer: r + 1,
6044                    agent: agent_id,
6045                    summary: String::new(),
6046                    findings: Vec::new(),
6047                    vote: None,
6048                    failed: None,
6049                    duration_ms: 0,
6050                    // Set for both outcomes: `failed: Some(_)` with
6051                    // `attempts > 0` is a seat every retry still lost, not a
6052                    // recovered one — only `failed: None` with `attempts > 0`
6053                    // reads as "answered after a nudge" (see this field's own
6054                    // doc).
6055                    attempts,
6056                };
6057                match res {
6058                    Ok((review, out)) => {
6059                        // Sanitized here, at the point every other piece of
6060                        // agent prose in this file is (candidate summaries,
6061                        // deliberation turns, vote reasons): a reviewer's own
6062                        // words are the one thing about it that could name
6063                        // it, and reconsideration below broadcasts this same
6064                        // summary and these same findings to every other
6065                        // seat on the panel.
6066                        record.summary =
6067                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6068                        record.vote = Some(review.vote);
6069                        record.duration_ms = out.duration_ms;
6070                        for (n, mut f) in review.findings.into_iter().enumerate() {
6071                            // ids are magi's, never the agent's: the fixer's
6072                            // adoption report is keyed by them.
6073                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6074                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6075                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6076                            // `file` is agent-supplied prose too, never
6077                            // checked against the real tree — the same
6078                            // exposure `title`/`detail` above have, just in
6079                            // a field easy to forget because it looks like a
6080                            // path rather than free text.
6081                            f.file = f
6082                                .file
6083                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6084                            all_findings.push(f.clone());
6085                            record.findings.push(f);
6086                        }
6087                        self.state.event(
6088                            "review",
6089                            format!(
6090                                "round {round}: reviewer {} voted {} with {} finding(s)",
6091                                r + 1,
6092                                review.vote.label(),
6093                                record.findings.len()
6094                            ),
6095                        );
6096                    }
6097                    Err(e) => {
6098                        record.failed = Some(e.to_string());
6099                        self.state.event(
6100                            "review",
6101                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6102                        );
6103                    }
6104                }
6105                records.push(record);
6106            }
6107
6108            // Tally the round's votes and, if they split, spend the one
6109            // round of reconsideration the split -> deliberate -> revote
6110            // shape `judge`/`vote` use for the panel, sized down to what a
6111            // read-only review round can afford: one round, and a revote
6112            // rather than an argument, because the panel already wrote its
6113            // reasoning down as findings the first time around.
6114            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6115            let vote_split =
6116                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6117            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6118            if vote_split {
6119                self.state.event(
6120                    "review",
6121                    format!(
6122                        "round {round}: votes split ({}) — one round of reconsideration",
6123                        initial_votes
6124                            .iter()
6125                            .map(|v| v.label())
6126                            .collect::<Vec<_>>()
6127                            .join(", ")
6128                    ),
6129                );
6130                // Seats read every seat's findings and votes, still numbered
6131                // and never named — the same anonymity `review` itself keeps.
6132                let panel: Vec<ReviewSeatReport<'_>> = records
6133                    .iter()
6134                    .filter_map(|r| {
6135                        r.vote.map(|vote| ReviewSeatReport {
6136                            reviewer: r.reviewer,
6137                            vote,
6138                            summary: &r.summary,
6139                            findings: &r.findings,
6140                        })
6141                    })
6142                    .collect();
6143
6144                let mut jobs = Vec::new();
6145                let mut seats_at = Vec::new();
6146                for (r, spec) in reviewers.iter().cloned().enumerate() {
6147                    // A seat with no initial vote has nothing to reconsider
6148                    // from and stays absent, the same as it stayed absent
6149                    // from `panel` above.
6150                    if records[r].vote.is_none() {
6151                        continue;
6152                    }
6153                    let wt = root.join(format!("review-{}", r + 1));
6154                    let seat_key = format!("review-{}", r + 1);
6155                    let spec = self.occupant(&seat_key, spec);
6156                    let seat = self.seat(&seat_key, &spec.id);
6157                    // A seat with no live session has already forgotten the
6158                    // initial review's prompt — restate the patch it is
6159                    // voting on, the same as `deliberate`/`vote` do for a
6160                    // judge in the same position.
6161                    // The panel already carries this seat's own review and
6162                    // vote, so restating the patch makes the prompt whole for
6163                    // a seat handed to another agent.
6164                    let build = |with_patch: bool| {
6165                        prompt::review_reconsider(&ReviewReconsiderCtx {
6166                            instruction: &self.state.instruction,
6167                            reviewer: r + 1,
6168                            lens: Lens::for_seat(r),
6169                            panel: &panel,
6170                            patch: with_patch.then_some(ReviewPatch {
6171                                branch: &winner.branch,
6172                                base_short: &base_short,
6173                                stat: &stat,
6174                                patch: &patch,
6175                            }),
6176                            round,
6177                            rounds: max_rounds,
6178                            language: &language,
6179                        })
6180                    };
6181                    let full = build(true);
6182                    let prompt = if has_context(&spec, &seat, sessions) {
6183                        build(false)
6184                    } else {
6185                        full.clone()
6186                    };
6187                    jobs.push(SeatJob {
6188                        prompt,
6189                        spec,
6190                        seat,
6191                        cwd: wt,
6192                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6193                        allow_write: false,
6194                        sessions,
6195                        artifacts: artifacts.clone(),
6196                        stem: format!("review-{round}-reconsider-{}", r + 1),
6197                        handover: Some(full),
6198                    });
6199                    seats_at.push(r);
6200                }
6201
6202                let mut recon_quota_losses = Vec::new();
6203                let recon_cache = self.state.config.cache_dir();
6204                let recon_ctx = WaveCtx {
6205                    carry_seats: true,
6206                    pauses: Some((&self.pause, &self.interrupt)),
6207                    run: &run_id,
6208                    node: "review",
6209                    prompts: &prompts,
6210                    cache: recon_cache.as_deref(),
6211                    round: Some(round),
6212                };
6213                let recon_results = ask_json_wave::<ReviewRevote>(
6214                    jobs,
6215                    Arc::clone(&self.sem),
6216                    review_retries,
6217                    &self.roles.reviewer_roster,
6218                    &recon_ctx,
6219                    &mut recon_quota_losses,
6220                    &mut self.state,
6221                    &|_: &ReviewRevote| Ok(()),
6222                )
6223                .await;
6224                if self.state.resource_block.is_some() {
6225                    return self.stop_for_resource();
6226                }
6227                self.state.quota.extend(recon_quota_losses);
6228
6229                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6230                    let agent_id = seat.agent.clone();
6231                    self.state.seats.insert(seat.key.clone(), seat);
6232                    let mut rec = ReviewRevoteRecord {
6233                        reviewer: r + 1,
6234                        agent: agent_id,
6235                        vote: None,
6236                        reason: String::new(),
6237                        failed: None,
6238                    };
6239                    match res {
6240                        Ok((rv, _)) => {
6241                            rec.vote = Some(rv.vote);
6242                            rec.reason =
6243                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6244                            self.state.event(
6245                                "review",
6246                                format!(
6247                                    "round {round}: reviewer {} revoted {}",
6248                                    r + 1,
6249                                    rv.vote.label()
6250                                ),
6251                            );
6252                        }
6253                        Err(e) => {
6254                            rec.failed = Some(e.to_string());
6255                            self.state.event(
6256                                "review",
6257                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6258                            );
6259                        }
6260                    }
6261                    reconsideration.push(rec);
6262                }
6263            } else if initial_votes.len() > 1 {
6264                self.state.event(
6265                    "review",
6266                    format!(
6267                        "round {round}: votes agreed ({}) — no reconsideration",
6268                        initial_votes[0].label()
6269                    ),
6270                );
6271            }
6272
6273            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6274            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6275            // A round that already has a blocking finding and a round left to
6276            // try is going back to the fixer no matter what `verify.e2e`
6277            // says, so running it first only spends the loop's slowest step
6278            // (minutes, for a Rust repo's full test suite) on a head about
6279            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6280            // runs once a round has no blocking findings left (see
6281            // `round_is_clean`, which a deferred — empty — `e2e` can never
6282            // satisfy since `blocking` is nonzero whenever this branch is
6283            // taken), and `stop_reviewing` forces a real run before it will
6284            // ever read a deferred round as green.
6285            let defer_e2e =
6286                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6287            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6288                let reason =
6289                    format!("{blocking} blocking finding(s) already required a fix this round");
6290                self.state.event(
6291                    "verify",
6292                    format!(
6293                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6294                         {}); it will run once a round has none left",
6295                        short(&head)
6296                    ),
6297                );
6298                (Vec::new(), false, true, Some(reason))
6299            } else {
6300                let e2e_commands = self.state.config.verify.e2e.clone();
6301                let cache_dir = self.state.config.cache_dir();
6302                let context = format!("round {round}");
6303                let wait = self.repository_wait();
6304                let (e2e, verify_retried) = with_cache_lease(
6305                    &mut self.state,
6306                    cache_dir.as_deref(),
6307                    "e2e",
6308                    "e2e",
6309                    &winner.worktree,
6310                    &head,
6311                    verify_timeout,
6312                    &context,
6313                    &wait,
6314                    |state, budget| {
6315                        let shell = shell.clone();
6316                        let e2e_commands = e2e_commands.clone();
6317                        let worktree = winner.worktree.clone();
6318                        let context = context.clone();
6319                        async move {
6320                            run_e2e_with_retry(
6321                                state,
6322                                &shell,
6323                                &e2e_commands,
6324                                &worktree,
6325                                budget,
6326                                &context,
6327                            )
6328                            .await
6329                        }
6330                    },
6331                )
6332                .await?;
6333                (e2e, verify_retried, false, None)
6334            };
6335
6336            let expected = records.len();
6337            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6338            let incomplete = answered < expected;
6339            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6340            let policy = self.state.config.graph.incomplete_review;
6341            let clean = round_is_clean(
6342                blocking,
6343                e2e_ok,
6344                answered,
6345                expected,
6346                round_quota_missing,
6347                policy,
6348            );
6349
6350            let mut round_record = ReviewRound {
6351                round,
6352                head: head.clone(),
6353                verified_head: None,
6354                verified_at: None,
6355                reviews: records,
6356                e2e,
6357                verify_retried,
6358                e2e_deferred,
6359                e2e_defer_reason,
6360                fix: None,
6361                blocking,
6362                answered,
6363                expected,
6364                clean,
6365                progressed: false,
6366                vote_split,
6367                reconsideration,
6368                verdict: None,
6369            };
6370            // The final vote per seat is its revote where reconsideration
6371            // ran and answered, its initial vote otherwise — the same
6372            // fallback `tally` uses for a judge whose private vote failed.
6373            round_record.verdict = ReviewVote::worst(
6374                round_record
6375                    .final_votes()
6376                    .into_iter()
6377                    .map(|(_, _, vote)| vote),
6378            );
6379            // Which commit and when magi actually attempted to check —
6380            // known the moment a command was dispatched against `head`,
6381            // whether or not it finished: a resource-blocked attempt still
6382            // targeted a specific commit at a specific time, and leaving
6383            // that unrecorded is exactly what made `verification_summary`
6384            // report a fresh attempt as "commit unknown ... recorded before
6385            // this was tracked", indistinguishable from a genuinely old,
6386            // untracked record. Only a deferred or unconfigured round never
6387            // ran at all and has nothing to record — see
6388            // `ReviewRound::verified_head`'s own doc.
6389            if !matches!(
6390                round_record.e2e_status(),
6391                E2eStatus::Deferred | E2eStatus::NotConfigured
6392            ) {
6393                round_record.verified_head = Some(head.clone());
6394                round_record.verified_at = Some(Timestamp::now());
6395            }
6396            let this_round_verification = round_record.verification_summary(&head);
6397
6398            if incomplete {
6399                let missing: Vec<String> = round_record
6400                    .reviews
6401                    .iter()
6402                    .filter(|r| r.failed.is_some())
6403                    .map(|r| format!("review-{}", r.reviewer))
6404                    .collect();
6405                self.state.event(
6406                    "review",
6407                    format!(
6408                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6409                        missing.join(", ")
6410                    ),
6411                );
6412            }
6413
6414            if clean {
6415                self.state.event(
6416                    "review",
6417                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6418                        format!(
6419                            "round {round}: clean (warn policy, incomplete panel) — no \
6420                             blocking findings from the seats that answered, verification green"
6421                        )
6422                    } else if incomplete {
6423                        format!(
6424                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6425                             quorum) — no blocking findings from the seats that answered, \
6426                             verification green",
6427                            expected - answered
6428                        )
6429                    } else {
6430                        format!("round {round}: clean — no blocking findings, verification green")
6431                    },
6432                );
6433                self.state.reviews.push(round_record);
6434                self.state.status = RunStatus::Gating;
6435                self.state.save()?;
6436                return Ok(());
6437            }
6438
6439            // Nothing was raised and verification passed, but not every seat
6440            // answered and `round_is_clean` still refused to call it clean —
6441            // either a seat is missing for a reason other than its own quota
6442            // (a crash, a timeout, unparsable output — worth another try), or
6443            // every seat that could have answered lost its quota and nobody
6444            // is left to decide on: re-review rather than send the fixer
6445            // after a round with nothing to fix.
6446            if incomplete && blocking == 0 && e2e_ok {
6447                self.state.reviews.push(round_record);
6448                self.state.save()?;
6449                if round == max_rounds {
6450                    self.state.status = RunStatus::Blocked;
6451                    self.state.event(
6452                        "review",
6453                        format!(
6454                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6455                             refusing to call it clean",
6456                            expected - answered
6457                        ),
6458                    );
6459                    return Ok(());
6460                }
6461                continue;
6462            }
6463
6464            // Nothing for the fixer to act on (`blocking == 0`) and the only
6465            // reason this round is not clean is that magi itself never got
6466            // a command to run — the shared build cache, not the patch (see
6467            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6468            // the fixer would invite a change to appease contention that has
6469            // nothing to do with the diff, and would leave this attempt
6470            // sitting in the next round's prompt as if it were about an
6471            // earlier, superseded commit rather than what it actually is:
6472            // the same head, still waiting to be checked. Wait for it the
6473            // same way the final round's own contention is already handled,
6474            // whatever round this happens to be.
6475            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6476                self.state.reviews.push(round_record);
6477                return self
6478                    .stop_reviewing(
6479                        "the round's own verification could not run",
6480                        &shell,
6481                        &winner.worktree,
6482                    )
6483                    .await;
6484            }
6485
6486            if round == max_rounds {
6487                self.state.reviews.push(round_record);
6488                return self
6489                    .stop_reviewing(
6490                        &format!(
6491                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6492                        ),
6493                        &shell,
6494                        &winner.worktree,
6495                    )
6496                    .await;
6497            }
6498
6499            // Fix. The winner's own implementer seat continues its conversation:
6500            // the competition is over, so context is pure benefit now.
6501            let blocking_findings: Vec<_> = all_findings
6502                .iter()
6503                .filter(|f| f.severity.blocks())
6504                .cloned()
6505                .collect();
6506            let fix_prompt = prompt::fix_in(
6507                &self.state.instruction,
6508                &blocking_findings,
6509                this_round_verification.as_ref(),
6510                round,
6511                max_rounds,
6512                &language,
6513                &self.state.config.graph.github_language,
6514            );
6515            let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6516            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6517            let (job, seat, out) = self
6518                .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6519                    prompt: fix_prompt.clone(),
6520                    spec,
6521                    seat,
6522                    cwd: winner.worktree.clone(),
6523                    timeout,
6524                    allow_write: true,
6525                    sessions,
6526                    artifacts: artifacts.clone(),
6527                    stem: format!("fix-{round}"),
6528                    handover: None,
6529                })
6530                .await;
6531            // Not asked: the round is not recorded (so a resume repeats it),
6532            // no fixer was handed over to, and the seat is left as it was.
6533            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6534                return self.stop_for_resource();
6535            }
6536            let agent_id = seat.agent.clone();
6537
6538            let mut fix = FixRecord {
6539                agent: agent_id,
6540                addressed: Vec::new(),
6541                rejected: Vec::new(),
6542                notes: String::new(),
6543                committed: false,
6544                failed: None,
6545                duration_ms: 0,
6546                continuation: None,
6547            };
6548            let mut continuation = ContinuationRecord::not_needed();
6549            let mut final_seat = seat.clone();
6550            match out {
6551                AgentOutcome::Ok(o) => {
6552                    fix.duration_ms = o.duration_ms;
6553                    let parsed = verdict::extract_json::<FixReport>(&o.text);
6554                    // A parsed report standing next to a command this same
6555                    // reply's own CLI never confirmed the exit status of is
6556                    // not a resolved answer — the identical `CommandEvidence`
6557                    // `state.jobs` renders, read here instead of only on
6558                    // display, per the completion judgment and the shown
6559                    // record needing to agree.
6560                    let incomplete_reason = match &parsed {
6561                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6562                            "the reply parsed, but it reported a command whose own CLI never \
6563                             confirmed an exit status"
6564                                .to_owned(),
6565                        ),
6566                        Ok(_) => None,
6567                        Err(e) => Some(e.to_string()),
6568                    };
6569                    match incomplete_reason {
6570                        None => {
6571                            let report = parsed.expect("checked Ok above");
6572                            fix.addressed = report.addressed;
6573                            fix.rejected = report.rejected;
6574                            fix.notes =
6575                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
6576                        }
6577                        Some(reason) => {
6578                            let (resumed_seat, resolved, failure, cont) = self
6579                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6580                                .await;
6581                            fix.duration_ms += cont.cumulative_wait_ms;
6582                            continuation = cont;
6583                            final_seat = resumed_seat;
6584                            match resolved {
6585                                Some(report) => {
6586                                    fix.addressed = report.addressed;
6587                                    fix.rejected = report.rejected;
6588                                    fix.notes = blind::sanitize_prose(
6589                                        &report.notes,
6590                                        &self.state.config.blind,
6591                                    );
6592                                }
6593                                None => fix.failed = failure,
6594                            }
6595                        }
6596                    }
6597                }
6598                // The CLI's raw error JSON is not a fix report to parse.
6599                AgentOutcome::Dropped(o) => {
6600                    fix.duration_ms = o.duration_ms;
6601                    let why = o
6602                        .dropped
6603                        .as_ref()
6604                        .map(|d| d.why.as_str())
6605                        .unwrap_or("the CLI ended the stream without delivering its answer");
6606                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6607                }
6608                AgentOutcome::Quota(o) => {
6609                    self.state.quota.push(QuotaLoss {
6610                        seat: final_seat.key.clone(),
6611                        node: "fix".to_owned(),
6612                        at: Timestamp::now(),
6613                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6614                    });
6615                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6616                }
6617                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6618            }
6619            fix.continuation = Some(continuation);
6620            self.state.seats.insert(final_seat.key.clone(), final_seat);
6621            let coordination = self.coordinate(&winner.worktree).await?;
6622            if let Ok(r) = git::rescue_commit(
6623                &winner.worktree,
6624                &format!("magi: review round {round} fixes (uncommitted work)"),
6625            )
6626            .await
6627            {
6628                self.state.note_withheld("fix", &r.withheld);
6629            }
6630            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6631            fix.committed = after != before;
6632            // Judged by what `git` says moved against base, never by the
6633            // fixer's own `addressed`/`rejected` count — see
6634            // `ReviewRound::progressed`. Propagated with `?`, the same as the
6635            // `patch` snapshot above: swallowing this error would default
6636            // `diff_after` to empty, which almost always differs from a
6637            // non-empty `patch` and reads as "progressed" — exactly backwards
6638            // for a `git` failure the stagnation check cannot see through.
6639            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6640            drop(coordination);
6641            let progressed = diff_after != patch;
6642            let commit_note = if fix.committed {
6643                "committed"
6644            } else {
6645                "NO new commit"
6646            };
6647            let tree_note = if progressed {
6648                "changed vs base"
6649            } else {
6650                "unchanged vs base"
6651            };
6652            self.state.event(
6653                "fix",
6654                match &fix.failed {
6655                    // Distinct on purpose from "0 addressed, 0 rejected": the
6656                    // fixer's own diff still landed (blocking counts do keep
6657                    // falling round over round), only its adoption report did
6658                    // not come back, so this must never read like every
6659                    // finding was reviewed and declined.
6660                    Some(reason) => {
6661                        format!(
6662                            "round {round}: fixer's adoption report was lost ({reason}); \
6663                             {commit_note}, tree {tree_note}"
6664                        )
6665                    }
6666                    None => format!(
6667                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6668                         {tree_note}{}",
6669                        fix.addressed.len(),
6670                        fix.rejected.len(),
6671                        if continuation.outcome == ContinuationOutcome::Resumed {
6672                            format!(
6673                                " (adoption report recovered after {} continuation(s))",
6674                                continuation.attempts
6675                            )
6676                        } else {
6677                            String::new()
6678                        },
6679                    ),
6680                },
6681            );
6682            round_record.fix = Some(fix);
6683            round_record.progressed = progressed;
6684            self.state.reviews.push(round_record);
6685            self.state.save()?;
6686
6687            // The fixer's own report never came back this round, even after
6688            // `continue_fix_report`'s own budget was spent on it — not an
6689            // ordinary "no report" (dropped stream, quota, plain failure),
6690            // which already reads that way and is left to the existing round
6691            // budget. Stopping here, rather than opening another round, is
6692            // what keeps a next reviewer/fixer wave from ever being
6693            // dispatched onto `winner.worktree` while whatever the seat's
6694            // last call may still have running there is unaccounted for: no
6695            // process liveness check exists (and none is being added — see
6696            // AGENTS.md/this task's own scope), so the only way to honour
6697            // "nothing starts before a valid report returns" is to not start
6698            // anything further on this worktree from this run at all.
6699            if matches!(
6700                continuation.outcome,
6701                ContinuationOutcome::Exhausted
6702                    | ContinuationOutcome::QuotaLost
6703                    | ContinuationOutcome::NoSession
6704            ) {
6705                return self
6706                    .stop_reviewing(
6707                        "the fixer's adoption report never came back, even after resuming its \
6708                         own seat; refusing to start another round against the same worktree \
6709                         while that is unresolved",
6710                        &shell,
6711                        &winner.worktree,
6712                    )
6713                    .await;
6714            }
6715
6716            let streak = self
6717                .state
6718                .reviews
6719                .iter()
6720                .rev()
6721                .take_while(|r| !r.progressed)
6722                .count();
6723            if streak >= STAGNANT_LIMIT {
6724                return self
6725                    .stop_reviewing(
6726                        &format!(
6727                            "the tree has not moved against base for {streak} round(s) in a row"
6728                        ),
6729                        &shell,
6730                        &winner.worktree,
6731                    )
6732                    .await;
6733            }
6734        }
6735        Ok(())
6736    }
6737
6738    /// Decide, from the last recorded round's own verification, whether
6739    /// stopping the review loop is a hand-off or a genuine block.
6740    ///
6741    /// Called once the loop has given up trying — the round budget is spent,
6742    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6743    /// findings still open, never while a round is still clean or the
6744    /// incomplete-panel case handled inline above. Gate and e2e are facts
6745    /// about the tree; a lingering review finding is an opinion, and this
6746    /// workload's own `magi stats` puts reviewer precision low enough
6747    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6748    /// must not by itself stand between a green, verified change and the
6749    /// human who decides what to do with it. A red e2e is not an opinion, so
6750    /// that case still blocks, with the failing command and a tail of its
6751    /// output recorded here rather than left in `run.json` for someone to go
6752    /// find.
6753    ///
6754    /// A round that deferred its own e2e (see [`Config::graph`]'s
6755    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6756    /// only because nothing ran, and treating an empty list as a passing one
6757    /// here is exactly the "deferred painted green" bug this function exists
6758    /// to not have. When the last round's own verification never resolved —
6759    /// deferred on purpose, or a real attempt the shared build cache blocked
6760    /// — this makes (or retries) the real run, on the actual worktree this
6761    /// loop is about to stop touching, before deciding anything. A
6762    /// resource-blocked attempt is likewise never read as either green or
6763    /// red: it is evidence about the machine, not the patch (see
6764    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6765    /// blocked cache leaves this call without deciding rather than guessing
6766    /// — the caller retries on a later reentry.
6767    /// Record, once, that the review loop handed off over a blocking finding
6768    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6769    /// `land` asks the owner even with `land_approval` off. Called from every
6770    /// path that concludes `Gating`; a reentry keeps the first record.
6771    fn record_contested_handoff(&mut self) {
6772        if self.state.contested_handoff.is_some() {
6773            return;
6774        }
6775        let Some(contested) = self
6776            .state
6777            .reviews
6778            .last()
6779            .and_then(ReviewRound::contested_handoff)
6780        else {
6781            return;
6782        };
6783        self.state.event(
6784            "review",
6785            format!(
6786                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6787                 wait for the owner's approval",
6788                contested.findings.len(),
6789                contested.rejecters.len()
6790            ),
6791        );
6792        self.state.contested_handoff = Some(contested);
6793    }
6794
6795    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6796        let round_idx = self.state.reviews.len() - 1;
6797        // A deferred round and a resource-blocked one are the same shape
6798        // here: neither has a real result yet, and both get one more
6799        // attempt. Read off `e2e_status` — the single source for this —
6800        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6801        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6802        // still retries instead of being read as a settled result the
6803        // instant it stops being empty.
6804        let needs_catchup_run = matches!(
6805            self.state.reviews[round_idx].e2e_status(),
6806            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6807        );
6808        if needs_catchup_run {
6809            let round = self.state.reviews[round_idx].round;
6810            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6811            let commands = self.state.config.verify.e2e.clone();
6812            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6813            let cache_dir = self.state.config.cache_dir();
6814            let context = format!(
6815                "round {round}: verification unresolved, catching up before the final decision"
6816            );
6817            let wait = self.repository_wait();
6818            let (outcomes, verify_retried) = with_cache_lease(
6819                &mut self.state,
6820                cache_dir.as_deref(),
6821                "e2e",
6822                "e2e",
6823                worktree,
6824                &attempted_head,
6825                timeout,
6826                &context,
6827                &wait,
6828                |state, budget| {
6829                    let shell = shell.to_vec();
6830                    let commands = commands.clone();
6831                    let context = context.clone();
6832                    async move {
6833                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6834                            .await
6835                    }
6836                },
6837            )
6838            .await?;
6839            let last = &mut self.state.reviews[round_idx];
6840            last.e2e = outcomes;
6841            last.verify_retried = verify_retried;
6842            // Always the commit and time this attempt actually targeted,
6843            // whether or not it happens to equal the reviewed `head` and
6844            // whether or not a command finished — see
6845            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6846            // attempt is recorded too, so a later reader sees "attempted
6847            // again at T2" rather than silence.
6848            last.verified_head = Some(attempted_head);
6849            last.verified_at = Some(Timestamp::now());
6850            if verify_inconclusive(&last.e2e) {
6851                // Still not a real result: `e2e_deferred` is left exactly
6852                // as it was, so `needs_catchup_run` above reads
6853                // `ResourceBlocked` (via `e2e_status`, which checks
6854                // `resource_blocked` before `e2e_deferred`) and retries
6855                // again on the next reentry, rather than recording
6856                // contention as a red e2e and blocking the run on it.
6857                self.state.save()?;
6858                return Ok(());
6859            }
6860            last.e2e_deferred = false;
6861        }
6862        let last = &self.state.reviews[round_idx];
6863        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6864
6865        match last.e2e_status() {
6866            E2eStatus::Failed => {
6867                let red: Vec<String> = last
6868                    .e2e
6869                    .iter()
6870                    .filter(|o| !o.ok())
6871                    .map(|o| {
6872                        format!(
6873                            "`{}` -> {:?}\n{}",
6874                            o.command,
6875                            o.code,
6876                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6877                        )
6878                    })
6879                    .collect();
6880                self.state
6881                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6882                self.state.status = RunStatus::Blocked;
6883            }
6884            // `needs_catchup_run` above already retried once this call; if
6885            // it is still blocked, this is magi's own admission it could
6886            // not get a command to run, never a verdict on the patch — the
6887            // run is left exactly where a later reentry can retry again.
6888            E2eStatus::ResourceBlocked => {
6889                self.state.event(
6890                    "review",
6891                    format!(
6892                        "{why}; e2e could not run (shared build cache unavailable); not \
6893                         deciding yet"
6894                    ),
6895                );
6896            }
6897            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6898                self.state.event(
6899                    "review",
6900                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6901                );
6902                self.record_contested_handoff();
6903                self.state.status = RunStatus::Gating;
6904            }
6905        }
6906        self.state.save()?;
6907        Ok(())
6908    }
6909
6910    // ----------------------------------------------------------------- gate
6911
6912    async fn gate(&mut self) -> Result<()> {
6913        // Judged by the review record itself, not by `status`: a solo
6914        // candidate's `judge`/`deliberate` skip rewrites `status` on every
6915        // reentry (see `judge`), and trusting it here is exactly how a run
6916        // that exhausted its review budget got gated and merged a second
6917        // time around. `review_conclusion` recomputes the review loop's own
6918        // verdict from the round records themselves — `Gating` for a clean
6919        // round or a hand-off (see `stop_reviewing`), anything else means the
6920        // loop is still going or genuinely blocked.
6921        // A base the winner could not be replayed onto is a decision, not a
6922        // round: there is no landing tree to gate. Read as its own record for
6923        // the same reason the review verdict is.
6924        if self.state.status == RunStatus::Failed
6925            || self
6926                .state
6927                .base_sync
6928                .as_ref()
6929                .is_some_and(|s| s.conflict.is_some())
6930            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6931                != Some(RunStatus::Gating)
6932        {
6933            return Ok(());
6934        }
6935        if self.state.gate_ran {
6936            // `review_loop` derives its conclusion from the clean review
6937            // record on every reentry and therefore puts a completed run back
6938            // in `Gating`. A recorded gate is a stronger, terminal fact:
6939            // retain its original command output (or lack of any, for a repo
6940            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
6941            // doc) and restore `Blocked` on a real failure rather than
6942            // pretending the command is still running or running it a second
6943            // time. `gate_ran == false` remains the only shape — unattempted,
6944            // or a resource-blocked retry — that may still need to execute a
6945            // command.
6946            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6947                self.state.status = RunStatus::Blocked;
6948                self.state.save()?;
6949            }
6950            return Ok(());
6951        }
6952        let Some(winner) = self.state.winner().cloned() else {
6953            return Ok(());
6954        };
6955        self.state.status = RunStatus::Gating;
6956        let mut outcomes = self.run_gate(&winner).await?;
6957        loop {
6958            // A resource-blocked outcome means the gate command never actually
6959            // ran - the shared build cache could not be acquired or confirmed
6960            // fresh in time - which is evidence about the machine, not about
6961            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
6962            // Recording it as a red gate would mark a run `Blocked` on nothing
6963            // but contention magi has already logged; leaving `self.state.gate`
6964            // empty and `self.state.gate_ran` false instead keeps the shape
6965            // this function already treats as "still needs to run" (see the
6966            // early-return above), so the next call retries the command
6967            // rather than concluding anything.
6968            if verify_inconclusive(&outcomes) {
6969                self.state.save()?;
6970                return Ok(());
6971            }
6972            if outcomes.iter().all(CommandOutcome::ok) {
6973                break;
6974            }
6975            match self.gate_fix_round(&winner, &outcomes).await? {
6976                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6977                GateFix::Stop => break,
6978                GateFix::Defer => {
6979                    self.state.save()?;
6980                    return Ok(());
6981                }
6982            }
6983        }
6984        let passed = outcomes.iter().all(CommandOutcome::ok);
6985        self.state.gate = outcomes;
6986        self.state.gate_ran = true;
6987        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6988        if !passed {
6989            self.state.status = RunStatus::Blocked;
6990            let spent = self.state.gate_fixes.len();
6991            self.state.event(
6992                "gate",
6993                if spent == 0 {
6994                    "gate failed; not merging".to_owned()
6995                } else {
6996                    format!("gate failed after {spent} gate-fix round(s); not merging")
6997                },
6998            );
6999        }
7000        self.state.save()?;
7001        Ok(())
7002    }
7003
7004    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7005    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7006    /// after review is clean and never on a candidate awaiting judging.
7007    ///
7008    /// Never fails the run: a non-zero exit or timeout is a warning and a
7009    /// recorded outcome, and the gate remains the single arbiter. Nothing
7010    /// configured means nothing happens - no event, no commit. `commit_all`
7011    /// commits any leftover change under the neutral identity and returns
7012    /// `false` when the tree is clean, so no empty commit is ever made.
7013    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7014        let commands = state.config.verify.pre_gate.clone();
7015        if commands.is_empty() {
7016            return;
7017        }
7018        let shell = state.config.shell();
7019        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7020        let (outcomes, _) = run_commands(
7021            state,
7022            "pre_gate",
7023            "pre_gate",
7024            0,
7025            &shell,
7026            &commands,
7027            &winner.worktree,
7028            timeout,
7029        )
7030        .await;
7031        for o in &outcomes {
7032            if !o.ok() {
7033                tracing::warn!(
7034                    "pre_gate `{}` failed ({:?}); the gate decides",
7035                    o.command,
7036                    o.code
7037                );
7038            }
7039            state.event(
7040                "pre_gate",
7041                format!(
7042                    "`{}` -> {}",
7043                    o.command,
7044                    if o.ok() {
7045                        "pass".to_owned()
7046                    } else {
7047                        format!(
7048                            "FAIL ({:?})\n{}",
7049                            o.code,
7050                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7051                        )
7052                    }
7053                ),
7054            );
7055        }
7056        state.pre_gate = outcomes;
7057        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7058            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7059                Ok(head) => {
7060                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7061                    state.pre_gate_commit = Some(head);
7062                }
7063                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7064            },
7065            Ok(false) => {}
7066            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7067        }
7068        if let Err(e) = state.save() {
7069            tracing::warn!("could not persist the pre_gate record: {e:#}");
7070        }
7071    }
7072
7073    /// Run `verify.gate` once against the winner's current tree, logging one
7074    /// event per command. Empty when nothing is configured.
7075    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7076        let wait = self.repository_wait();
7077        run_landing_gate(&mut self.state, winner, &wait).await
7078    }
7079
7080    /// One bounded fix round for a failing gate.
7081    ///
7082    /// The fixer is told the failure came from the gate itself, not from a
7083    /// reviewer, and is shown the failed commands, their exit codes and a tail
7084    /// of their output - whatever `[verify].gate` holds, nothing here knows
7085    /// what those commands run. Only a normal non-zero exit that printed
7086    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7087    /// command or a full disk says nothing about the code, and a fixer sent
7088    /// after it can only appease the machine. The round is judged by what git
7089    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7090    /// again before the gate does, so a fix cannot trade a green gate for a
7091    /// red e2e unnoticed.
7092    async fn gate_fix_round(
7093        &mut self,
7094        winner: &Candidate,
7095        outcomes: &[CommandOutcome],
7096    ) -> Result<GateFix> {
7097        let cap = self.state.config.graph.gate_fix_rounds;
7098        let spent = self.state.gate_fixes.len();
7099        if spent >= cap {
7100            if cap > 0 {
7101                self.state.event(
7102                    "gate",
7103                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7104                );
7105            }
7106            return Ok(GateFix::Stop);
7107        }
7108        if !gate_fixable(outcomes) {
7109            self.state.event(
7110                "gate",
7111                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7112                 command or similar); not spending a fix round on it",
7113            );
7114            return Ok(GateFix::Stop);
7115        }
7116        let min_free = self.state.config.disk.min_free_bytes;
7117        if min_free > 0 {
7118            match crate::disk::free_bytes(&winner.worktree) {
7119                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7120                Ok(free) => {
7121                    self.state.event(
7122                        "gate",
7123                        format!(
7124                            "only {free} bytes free ({min_free} required by `[disk] \
7125                             min_free_bytes`); not spending a fix round on a failure the disk \
7126                             may explain"
7127                        ),
7128                    );
7129                    return Ok(GateFix::Stop);
7130                }
7131                Err(e) => {
7132                    self.state.event(
7133                        "gate",
7134                        format!("free disk space could not be measured ({e:#}); no fix round"),
7135                    );
7136                    return Ok(GateFix::Stop);
7137                }
7138            }
7139        }
7140
7141        let attempt = spent + 1;
7142        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7143        let base = self.landing_base();
7144        let fix_prompt = prompt::gate_fix_in(
7145            &self.state.instruction,
7146            &failed,
7147            attempt,
7148            cap,
7149            &self.state.config.graph.language,
7150            &self.state.config.graph.github_language,
7151        );
7152        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7153        let sessions = self.state.config.graph.sessions;
7154        let artifacts = self.merge_repair_artifacts();
7155        self.state.event(
7156            "gate",
7157            format!("gate failed; gate-fix round {attempt} of {cap}"),
7158        );
7159        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7160        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7161        // Persist before invoking the fixer: it may commit and then be
7162        // interrupted before its result (or the following e2e) is saved.
7163        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7164            head: before.clone(),
7165            passed: false,
7166            outcomes: Vec::new(),
7167        });
7168        self.state.save()?;
7169        let (_, seat, out) = self
7170            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7171                prompt: fix_prompt.clone(),
7172                spec,
7173                seat,
7174                cwd: winner.worktree.clone(),
7175                timeout,
7176                allow_write: true,
7177                sessions,
7178                artifacts: artifacts.clone(),
7179                stem: format!("gate-fix-{attempt}"),
7180                handover: None,
7181            })
7182            .await;
7183        // The fixer was never asked: no round is spent (nothing is recorded)
7184        // and a later reentry asks the same agent again.
7185        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7186            return Ok(GateFix::Defer);
7187        }
7188        let mut record = GateFixRecord {
7189            agent: seat.agent.clone(),
7190            failed,
7191            notes: String::new(),
7192            committed: false,
7193            error: None,
7194        };
7195        match out {
7196            AgentOutcome::Ok(o) => {
7197                // A missing report is not a failed fix: the round is judged
7198                // by the tree below, and the report only carries prose.
7199                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7200                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7201                }
7202            }
7203            AgentOutcome::Dropped(_) => {
7204                record.error = Some("the CLI dropped the stream".to_owned());
7205            }
7206            AgentOutcome::Quota(o) => {
7207                self.state.quota.push(QuotaLoss {
7208                    seat: seat.key.clone(),
7209                    node: "gate-fix".to_owned(),
7210                    at: Timestamp::now(),
7211                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7212                });
7213                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7214            }
7215            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7216        }
7217        self.state.seats.insert(seat.key.clone(), seat);
7218        let coordination = self.coordinate(&winner.worktree).await?;
7219        if let Ok(r) = git::rescue_commit(
7220            &winner.worktree,
7221            &format!("magi: gate fix {attempt} (uncommitted work)"),
7222        )
7223        .await
7224        {
7225            self.state.note_withheld("gate-fix", &r.withheld);
7226        }
7227        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7228        record.committed = after != before;
7229        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7230        let note = record.error.clone();
7231        self.state.gate_fixes.push(record);
7232        self.state.save()?;
7233        if !changed {
7234            self.state.event(
7235                "gate",
7236                match note {
7237                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7238                    None => format!("gate-fix round {attempt}: the tree did not change"),
7239                },
7240            );
7241            return Ok(GateFix::Stop);
7242        }
7243        self.state.event(
7244            "gate",
7245            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7246        );
7247
7248        drop(coordination);
7249        let wait = self.repository_wait();
7250        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7251    }
7252
7253    // ---------------------------------------------------------------- merge
7254
7255    /// One read-only rewrite by the summary's author; a text the gate still
7256    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7257    /// applies on their word or on silence. `None` means the run is parked on
7258    /// that question and nothing may be pushed or opened yet.
7259    async fn guarded_pr_message(
7260        &mut self,
7261        winner: &Candidate,
7262        facts: Option<&BranchFacts>,
7263        posting: bool,
7264    ) -> Result<Option<PrMessage>> {
7265        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7266        let head = match final_head.as_deref() {
7267            Some(h) => crate::pr_outcome::FinalHead::At(h),
7268            None => crate::pr_outcome::FinalHead::Unknown,
7269        };
7270        // Only a pull request that is about to be posted pays for a summary.
7271        let summarized = if posting {
7272            Some(self.summarize_change(winner, final_head.as_deref()).await)
7273        } else {
7274            None
7275        };
7276        // A busy shared resource is not a permanent cause: publishing an
7277        // "unavailable" description would stick (a later head move never
7278        // edits the body). Stop before anything is pushed or opened; the
7279        // run is `Blocked` and a resume asks the summary again.
7280        if self.state.resource_block.is_some() {
7281            self.state.event(
7282                "pr-summary",
7283                "a shared resource was busy; nothing was published, resume to summarize again",
7284            );
7285            return Ok(None);
7286        }
7287        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7288            anyhow::bail!(
7289                "the final commit moved during public description generation; retry before publishing"
7290            );
7291        }
7292        let note = match (&summarized, &self.state.change_summary) {
7293            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7294            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7295            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7296            (None, _) => ChangeNote::NotAsked,
7297        };
7298        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7299        if !posting {
7300            let identity = crate::scrub::Identity::current();
7301            return Ok(Some(PrMessage {
7302                title: crate::scrub::scrub(&pr.title, &identity),
7303                body: crate::scrub::scrub(&pr.body, &identity),
7304            }));
7305        }
7306        // This very text was already rejected and asked about: no second
7307        // judge call, rewrite or question, just carry the record on.
7308        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7309        if let Some(known) = self
7310            .state
7311            .github_text
7312            .clone()
7313            .filter(|g| g.fingerprint == raw_fp)
7314        {
7315            return self.settle_withheld(winner, &pr, known).await;
7316        }
7317        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7318        let mut violations =
7319            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7320        let identity = crate::scrub::Identity::current();
7321        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7322            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7323            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7324        {
7325            violations.push(crate::github_text::Violation::SensitiveData);
7326        }
7327        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7328        // only a language violation starts one; sensitive data alone goes
7329        // straight to the owner, with the redacted text as the fallback.
7330        let has_language = violations
7331            .iter()
7332            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7333        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7334            self.state.event(
7335                "github-text",
7336                if has_language {
7337                    format!("description rejected: {violations:?}; requesting one rewrite")
7338                } else {
7339                    format!("description rejected: {violations:?}; asking the owner")
7340                },
7341            );
7342            let mut rewritten = false;
7343            let mut rejected_rewrite = None;
7344            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7345            {
7346                let output = text;
7347                if let Some(title) = summary_title(&output) {
7348                    // Judge the redacted candidate that will actually be posted.
7349                    // Sensitive data alone must not discard a useful rewrite.
7350                    let raw_body = summary_without_title(&output);
7351                    let clean_title = crate::scrub::scrub(&title, &identity);
7352                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7353                    let redacted = clean_title != title || body != raw_body;
7354                    let title = clean_title;
7355                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7356                    if !complete {
7357                        self.state.event(
7358                            "github-text",
7359                            "description rewrite omitted required findings",
7360                        );
7361                    }
7362                    if !body.trim().is_empty() {
7363                        body.push_str("\n\n");
7364                        body.push_str(&public_pr_evidence(&self.state, head));
7365                    }
7366                    let marker = format!("magi:run/{}", self.state.id);
7367                    if !body.trim().is_empty() && !body.contains(&marker) {
7368                        body.push_str(&format!("\n\n{marker}"));
7369                    }
7370                    let rewrite = PrMessage {
7371                        title: title.clone(),
7372                        body: body.clone(),
7373                    };
7374                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7375                    let mut failed = crate::github_text::check_for(
7376                        &self.state.config.graph,
7377                        &title,
7378                        &body,
7379                        redecision,
7380                    );
7381                    if (!crate::github_text::shareable(&title)
7382                        || !crate::github_text::shareable(&body))
7383                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7384                    {
7385                        failed.push(crate::github_text::Violation::SensitiveData);
7386                    }
7387                    if complete
7388                        && !title.trim().is_empty()
7389                        && !body.trim().is_empty()
7390                        && !failed.is_empty()
7391                    {
7392                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7393                            title: title.clone(),
7394                            body: body.clone(),
7395                            failed: failed
7396                                .iter()
7397                                .map(|v| crate::github_text::category(*v).to_owned())
7398                                .collect(),
7399                        });
7400                    }
7401                    if complete
7402                        && !title.trim().is_empty()
7403                        && !raw_body.trim().is_empty()
7404                        && failed.is_empty()
7405                    {
7406                        decision = redecision;
7407                        pr = PrMessage { title, body };
7408                        rewritten = true;
7409                        self.state
7410                            .event("github-text", "description rewrite passed");
7411                        if redacted {
7412                            self.state.event(
7413                                "github-text",
7414                                "sensitive data removed from description rewrite",
7415                            );
7416                        }
7417                    }
7418                }
7419            }
7420            if !rewritten {
7421                self.state.event(
7422                    "github-text",
7423                    "description rewrite unavailable or rejected; asking the owner",
7424                );
7425                if let Some(w) =
7426                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7427                {
7428                    // A different text than the one asked about before: that
7429                    // question no longer describes anything.
7430                    self.retire_github_text_question("the withheld text changed");
7431                    let record = crate::run::GithubTextAsk {
7432                        rewrite: rejected_rewrite,
7433                        fingerprint: raw_fp,
7434                        title: w.title,
7435                        body: w.body,
7436                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7437                        question: None,
7438                        asks: 0,
7439                        resolved: false,
7440                        chosen_title: None,
7441                        choice: None,
7442                    };
7443                    return self.settle_withheld(winner, &pr, record).await;
7444                }
7445            }
7446        }
7447        let (title, body) =
7448            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7449        Ok(Some(PrMessage { title, body }))
7450    }
7451
7452    /// Describe the whole change from its final diff in either run mode.
7453    ///
7454    /// The summary is tied to the final commit and to the merge base of the
7455    /// resolved remote base; a record for any other pair is dropped and the
7456    /// agent asked again, so a description never outlives the diff it was
7457    /// written from. The seat is read-only and the prompt tells it to read the
7458    /// diff and the code around it. `[roles] synthesizer` selects the agents
7459    /// (unset: the default pick), each asked at most once, advancing on an
7460    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7461    /// carries the reason it is unavailable; nothing is invented then.
7462    async fn summarize_change(
7463        &mut self,
7464        winner: &Candidate,
7465        head: Option<&str>,
7466    ) -> std::result::Result<(), String> {
7467        let Some(head) = head else {
7468            return Err("the final commit could not be read".to_owned());
7469        };
7470        let remote = self.state.config.merge.remote.clone();
7471        let tracking = format!("{remote}/{}", self.state.base_branch);
7472        if !git::rev_exists(&winner.worktree, &tracking).await {
7473            return Err(format!("the remote base `{tracking}` could not be read"));
7474        }
7475        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7476            Ok(b) if !b.is_empty() => b,
7477            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7478        };
7479        let evidence = crate::pr_outcome::summary_evidence(
7480            &self.state,
7481            crate::pr_outcome::FinalHead::At(head),
7482        );
7483        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7484        if self
7485            .state
7486            .change_summary
7487            .as_ref()
7488            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7489        {
7490            return Ok(());
7491        }
7492        if self.state.change_summary.take().is_some() {
7493            self.state.event(
7494                "pr-summary",
7495                "the final diff or evidence changed; the earlier change summary was discarded",
7496            );
7497        }
7498        // Save the source snapshot even if no generation agent is available.
7499        if let Err(e) = write_artifact(
7500            &self.state,
7501            &format!("public-description-{fingerprint}.md"),
7502            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7503        ) {
7504            tracing::warn!("could not save public description evidence: {e:#}");
7505        }
7506        let chain = match agent::pick_chain(
7507            &self.state.config.agents,
7508            self.state.config.roles.synthesizer.as_ref(),
7509            &agent::installed,
7510            "synthesizer",
7511        ) {
7512            Ok(chain) => chain,
7513            Err(e) => {
7514                tracing::warn!("no agent for the change summary: {e:#}");
7515                self.state.event("pr-summary", "no summary agent available");
7516                return Err("no agent was available to summarize it".to_owned());
7517            }
7518        };
7519        let subjects = git::subjects(&winner.worktree, &base, head)
7520            .await
7521            .unwrap_or_default();
7522        let stat = git::diff_stat(&winner.worktree, &base, head)
7523            .await
7524            .unwrap_or_default();
7525        let stat: String = stat.chars().take(6_000).collect();
7526        let mut prompt = crate::prompt::change_summary_in(
7527            &base,
7528            head,
7529            &subjects,
7530            stat.trim(),
7531            &self.state.config.graph.github_language,
7532        );
7533        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7534        let sessions = self.state.config.graph.sessions;
7535        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7536        let prompts = self.state.config.prompts.clone();
7537        let cache = self.state.config.cache_dir();
7538        let run = self.state.id.clone();
7539        for (i, spec) in chain.into_iter().enumerate() {
7540            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7541            let job = SeatJob {
7542                spec: spec.clone(),
7543                seat,
7544                cwd: winner.worktree.clone(),
7545                prompt: prompt.clone(),
7546                timeout,
7547                allow_write: false,
7548                sessions,
7549                artifacts: agent::artifacts_dir(&self.state.dir()),
7550                stem: if i == 0 {
7551                    "change-summary".to_owned()
7552                } else {
7553                    format!("change-summary-{}", spec.id)
7554                },
7555                handover: None,
7556            };
7557            let ctx = WaveCtx {
7558                run: &run,
7559                node: "pr-summary",
7560                prompts: &prompts,
7561                cache: cache.as_deref(),
7562                round: None,
7563                carry_seats: false,
7564                pauses: Some((&self.pause, &self.interrupt)),
7565            };
7566            let (seat, outcome) =
7567                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7568            self.state.seats.insert(seat.key.clone(), seat);
7569            let text = match outcome {
7570                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7571                AgentOutcome::ResourceBlocked(why) => {
7572                    // No agent was asked: the run stops resumably before
7573                    // anything is published (see `guarded_pr_message`).
7574                    if self.state.resource_block.is_none() {
7575                        self.state.resource_block = Some(crate::run::ResourceBlock {
7576                            node: "pr-summary".to_owned(),
7577                            kind: "shared resource".to_owned(),
7578                            detail: why,
7579                            at: Timestamp::now(),
7580                        });
7581                    }
7582                    return Err("a shared resource was busy".to_owned());
7583                }
7584                _ => continue,
7585            };
7586            let blind = &self.state.config.blind;
7587            let Some(title) = summary_title(&text) else {
7588                continue;
7589            };
7590            let body: String = summary_without_title(&text)
7591                .lines()
7592                .filter(|l| !l.contains("magi:run/"))
7593                .collect::<Vec<_>>()
7594                .join("\n");
7595            let body = blind::sanitize_prose(body.trim(), blind);
7596            let title = blind::sanitize_prose(&title, blind);
7597            if body.trim().is_empty() || title.trim().is_empty() {
7598                continue;
7599            }
7600            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7601                continue;
7602            }
7603            // The branch must still be the one the agent was shown.
7604            if git::rev_parse(&winner.worktree, "HEAD")
7605                .await
7606                .ok()
7607                .as_deref()
7608                != Some(head)
7609            {
7610                return Err("the branch moved while the summary was written".to_owned());
7611            }
7612            if git::merge_base(&winner.worktree, &tracking, head)
7613                .await
7614                .ok()
7615                .as_deref()
7616                != Some(base.as_str())
7617                || crate::pr_outcome::summary_evidence(
7618                    &self.state,
7619                    crate::pr_outcome::FinalHead::At(head),
7620                ) != evidence
7621            {
7622                return Err("the summary evidence changed while it was written".to_owned());
7623            }
7624            self.state.change_summary = Some(crate::run::ChangeSummary {
7625                head: head.to_owned(),
7626                base,
7627                evidence: fingerprint,
7628                title,
7629                body,
7630            });
7631            self.state.event(
7632                "pr-summary",
7633                format!("change summary written by {} for the final diff", spec.id),
7634            );
7635            if let Err(e) = self.state.save() {
7636                tracing::warn!("could not save the change summary: {e:#}");
7637            }
7638            return Ok(());
7639        }
7640        self.state
7641            .event("pr-summary", "no agent produced a change summary");
7642        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7643    }
7644
7645    /// One read-only rewrite request for a withheld description. The
7646    /// implementer's own conversation is reused only when it exists and can be
7647    /// resumed; otherwise the `[roles] language_judge` chain (or the default
7648    /// pick) writes it in a separate conversation. The prompt carries the whole
7649    /// text, so no agent needs any memory. Each agent is asked at most once and
7650    /// the chain advances on an error, quota, unusable answer or a reply with
7651    /// no `TITLE:` line. Nothing is recorded as a handover.
7652    async fn rewrite_pr_text(
7653        &mut self,
7654        winner: &Candidate,
7655        pr: &PrMessage,
7656        violations: &[crate::github_text::Violation],
7657    ) -> Option<String> {
7658        let prompt = format!(
7659            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7660            pr.title, pr.body
7661        );
7662        let sessions = self.state.config.graph.sessions;
7663        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7664        // (spec, seat, resumed)
7665        let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7666        let key = format!("impl-{}", winner.label);
7667        if let Some(spec) = self
7668            .state
7669            .config
7670            .agents
7671            .iter()
7672            .find(|a| a.id == winner.agent)
7673            .cloned()
7674            && let Some(existing) = self.state.seats.get(&key)
7675            && existing.agent == spec.id
7676            && agent::has_session(spec.kind, existing, sessions)
7677        {
7678            attempts.push((spec, existing.clone(), true));
7679        } else {
7680            let chain = agent::pick_chain(
7681                &self.state.config.agents,
7682                self.state.config.roles.language_judge.as_ref(),
7683                &agent::installed,
7684                "language_judge",
7685            );
7686            match chain {
7687                Ok(chain) => {
7688                    for spec in chain {
7689                        let seat = crate::fixer::seat_for(
7690                            &mut self.state,
7691                            "github-text-rewrite",
7692                            &spec.id,
7693                        );
7694                        attempts.push((spec, seat, false));
7695                    }
7696                }
7697                Err(e) => {
7698                    tracing::warn!("no agent for the description rewrite: {e:#}");
7699                    self.state
7700                        .event("github-text", "no rewrite agent available");
7701                    return None;
7702                }
7703            }
7704        }
7705        let prompts = self.state.config.prompts.clone();
7706        let cache = self.state.config.cache_dir();
7707        let run = self.state.id.clone();
7708        for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7709            let job = SeatJob {
7710                spec: spec.clone(),
7711                seat,
7712                cwd: winner.worktree.clone(),
7713                prompt: prompt.clone(),
7714                timeout: retry_budget(timeout_full, resumed),
7715                allow_write: false,
7716                sessions,
7717                artifacts: agent::artifacts_dir(&self.state.dir()),
7718                stem: if i == 0 {
7719                    "github-text-rewrite".to_owned()
7720                } else {
7721                    format!("github-text-rewrite-{}", spec.id)
7722                },
7723                handover: None,
7724            };
7725            let ctx = WaveCtx {
7726                run: &run,
7727                node: "github-text",
7728                prompts: &prompts,
7729                cache: cache.as_deref(),
7730                round: None,
7731                carry_seats: false,
7732                pauses: Some((&self.pause, &self.interrupt)),
7733            };
7734            let (seat, outcome) =
7735                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7736            self.state.seats.insert(seat.key.clone(), seat);
7737            match outcome {
7738                AgentOutcome::Ok(o)
7739                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7740                {
7741                    return Some(o.text);
7742                }
7743                AgentOutcome::ResourceBlocked(_) => return None,
7744                _ => {}
7745            }
7746        }
7747        None
7748    }
7749
7750    /// Abandon the standing posting-gate question, if any.
7751    fn retire_github_text_question(&mut self, why: &str) {
7752        let Some(id) = self
7753            .state
7754            .github_text
7755            .as_ref()
7756            .and_then(|g| g.question.clone())
7757        else {
7758            return;
7759        };
7760        let _ = ask::Questions::open().update(&id, |q| {
7761            q.abandon(why);
7762            Ok(())
7763        });
7764    }
7765
7766    /// Carry a withheld text to its end: file the question, park while it is
7767    /// open, and on an answer, silence or timeout pick the text to post. The
7768    /// decision is saved before it is used, so it applies once.
7769    async fn settle_withheld(
7770        &mut self,
7771        winner: &Candidate,
7772        pr: &PrMessage,
7773        mut g: crate::run::GithubTextAsk,
7774    ) -> Result<Option<PrMessage>> {
7775        use crate::github_text as gt;
7776        let store = ask::Questions::open();
7777        let timeout = self.state.config.graph.answer_timeout;
7778        if !g.resolved {
7779            let mut standing = g
7780                .question
7781                .as_ref()
7782                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7783            if let Some(q) = standing.as_ref()
7784                && q.status.open()
7785                && gt::expired(q, timeout)
7786            {
7787                let why = format!("no answer within {}s of asking", timeout.max(1));
7788                standing = store
7789                    .update(&q.id, |q| {
7790                        q.abandon(&why);
7791                        Ok(())
7792                    })
7793                    .ok()
7794                    .map(|(q, ())| q);
7795            }
7796            let mut ask_again: Option<Option<Vec<String>>> = None;
7797            match standing {
7798                // Recorded but never written (a stop between the two saves),
7799                // or removed: ask again without spending the retry.
7800                None if g.asks <= 1 => {
7801                    g.asks = 0;
7802                    ask_again = Some(None);
7803                }
7804                None => {
7805                    g.resolved = true;
7806                    g.chosen_title = None;
7807                    g.choice = Some(GithubTextChoice::Fallback);
7808                }
7809                Some(q) if q.status.open() => {
7810                    self.park_on_github_text(&g)?;
7811                    return Ok(None);
7812                }
7813                Some(q) => match gt::read_reply(&q) {
7814                    gt::Reply::Fallback => {
7815                        g.resolved = true;
7816                        g.chosen_title = None;
7817                        g.choice = Some(GithubTextChoice::Fallback);
7818                    }
7819                    gt::Reply::Original => {
7820                        g.resolved = true;
7821                        g.chosen_title = None;
7822                        g.choice = Some(GithubTextChoice::Original);
7823                    }
7824                    gt::Reply::Rewrite => {
7825                        g.resolved = true;
7826                        g.chosen_title = None;
7827                        g.choice = Some(if g.rewrite.is_some() {
7828                            GithubTextChoice::Rewrite
7829                        } else {
7830                            GithubTextChoice::Fallback
7831                        });
7832                    }
7833                    gt::Reply::Title(text) => {
7834                        let first = text
7835                            .lines()
7836                            .map(str::trim)
7837                            .find(|l| !l.is_empty())
7838                            .unwrap_or("");
7839                        let decision = if first.is_empty() {
7840                            None
7841                        } else {
7842                            let candidate = PrMessage {
7843                                title: first.to_owned(),
7844                                body: String::new(),
7845                            };
7846                            self.judge_pr_language(&winner.worktree, &candidate).await
7847                        };
7848                        match gt::vet_title(
7849                            &text,
7850                            decision,
7851                            self.state.config.graph.github_english(),
7852                        ) {
7853                            Ok(title) => {
7854                                g.resolved = true;
7855                                g.chosen_title = Some(title);
7856                                g.choice = Some(GithubTextChoice::MyTitle);
7857                            }
7858                            Err(cats) if g.asks < 2 => {
7859                                ask_again = Some(Some(
7860                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7861                                ))
7862                            }
7863                            Err(_) => {
7864                                g.resolved = true;
7865                                g.chosen_title = None;
7866                                g.choice = Some(GithubTextChoice::Fallback);
7867                            }
7868                        }
7869                    }
7870                },
7871            }
7872            if let Some(rejected) = ask_again {
7873                // A retry (after a rejected replacement title) is still about
7874                // the original text: the same ORIGINAL / FALLBACK comparison.
7875                let w = gt::Withheld {
7876                    title: g.title,
7877                    body: g.body,
7878                    categories: g.categories.clone(),
7879                };
7880                // Named by the original text's fingerprint alone, so a resume
7881                // rewrites the same file and the detail stays identical.
7882                let artifact = match crate::run::write_artifact(
7883                    &self.state,
7884                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
7885                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7886                ) {
7887                    Ok(p) => Some(p),
7888                    Err(e) => {
7889                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
7890                        None
7891                    }
7892                };
7893                let (detail, comparison) = gt::question_content(
7894                    &w,
7895                    &pr.title,
7896                    &pr.body,
7897                    &self.state.id,
7898                    rejected.as_deref(),
7899                    artifact.as_deref(),
7900                    g.rewrite.as_ref(),
7901                );
7902                let mut q = ask::Question::new(
7903                    self.state.id.clone(),
7904                    gt::ASK_NODE.to_owned(),
7905                    gt::ASK_SEAT.to_owned(),
7906                    gt::question_summary(&self.state.config.graph.language, &w),
7907                    detail,
7908                    gt::question_choices(&w, g.rewrite.as_ref()),
7909                );
7910                q.posting_comparison = Some(comparison);
7911                q.answer_timeout = timeout;
7912                // Recorded before it is written: a stop in between leaves a
7913                // record naming a question the resume then files again.
7914                g.question = Some(q.id.clone());
7915                g.asks += 1;
7916                self.state.github_text = Some(g.clone());
7917                self.state.event(
7918                    "github-text",
7919                    format!("asking the owner about the withheld text ({})", q.short()),
7920                );
7921                self.state.save()?;
7922                store
7923                    .put_executable(&mut q)
7924                    .context("file the posting-gate question")?;
7925                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7926                    tracing::warn!(
7927                        "could not notify about posting-gate question {}: {e:#}",
7928                        q.short()
7929                    );
7930                }
7931                self.park_on_github_text(&g)?;
7932                return Ok(None);
7933            }
7934        }
7935        let resolved = g.chosen_title.clone();
7936        let original = g.choice == Some(GithubTextChoice::Original);
7937        // Field names only: the event log never records the text.
7938        self.state.event(
7939            "github-text",
7940            if original {
7941                format!(
7942                    "posting the original text for: {}",
7943                    match (g.title, g.body) {
7944                        (true, true) => "title, description",
7945                        (true, false) => "title",
7946                        _ => "description",
7947                    }
7948                )
7949            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7950                "posting the saved rewrite title and description".to_owned()
7951            } else if resolved.is_some() {
7952                "posting the owner's replacement title".to_owned()
7953            } else {
7954                "posting the neutral text for the withheld fields".to_owned()
7955            },
7956        );
7957        self.state.github_text = Some(g.clone());
7958        self.state.save()?;
7959        if g.choice == Some(GithubTextChoice::Rewrite)
7960            && let Some(r) = g.rewrite.as_ref()
7961        {
7962            return Ok(Some(PrMessage {
7963                title: r.title.clone(),
7964                body: r.body.clone(),
7965            }));
7966        }
7967        let w = gt::Withheld {
7968            title: g.title,
7969            body: g.body,
7970            categories: g.categories.clone(),
7971        };
7972        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7973        if let Some(t) = resolved {
7974            title = t;
7975        }
7976        // Every withheld field is now neutral or vetted English.
7977        let english = gt::LanguageDecision {
7978            title_english: true,
7979            body_english: true,
7980            body_complete: true,
7981        };
7982        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7983        if original {
7984            // The owner's explicit choice overrides the gate for the fields it
7985            // was made for; the others keep what the gate produced.
7986            if g.title {
7987                title = pr.title.clone();
7988            }
7989            if g.body {
7990                body = pr.body.clone();
7991            }
7992        }
7993        Ok(Some(PrMessage { title, body }))
7994    }
7995
7996    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7997        self.state.github_text = Some(g.clone());
7998        self.state.end_pass_parked(ParkCause::Question);
7999        self.state.parked = true;
8000        self.state.event(
8001            "github-text",
8002            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8003        );
8004        self.state.save()
8005    }
8006
8007    /// Ask `[roles] language_judge` about this text and record which source
8008    /// decided; the text itself is never recorded.
8009    async fn judge_pr_language(
8010        &mut self,
8011        cwd: &Path,
8012        pr: &PrMessage,
8013    ) -> Option<crate::github_text::LanguageDecision> {
8014        self.state.config.roles.language_judge.as_ref()?;
8015        if !self.state.config.graph.language_gate() {
8016            return None;
8017        }
8018        let decision =
8019            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8020        self.state.event(
8021            "github-text",
8022            if decision.is_some() {
8023                "language decided by the language judge"
8024            } else {
8025                "language judge unavailable; using built-in heuristics"
8026            },
8027        );
8028        decision
8029    }
8030
8031    async fn merge(&mut self) -> Result<()> {
8032        // Same reasoning as `gate`: ask the review and gate records directly
8033        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8034        // skip can rewrite on reentry to something that no longer says
8035        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8036        // so a hand-off (open findings, green verification) reaches merge
8037        // exactly like a genuinely clean round does.
8038        //
8039        // A run resumed mid-`land` never reaches here at all: `execute`
8040        // recognises `RunStatus::Landing` before it even calls `prep`, and
8041        // routes straight to `run_land` instead. That has to happen a level
8042        // up from this function, not with a check in here, because
8043        // `review_loop`'s own status recomputation (see its doc) runs
8044        // *before* `merge` on every reentry and would otherwise overwrite
8045        // the `Landing` marker with `Gating` before this node ever saw it.
8046        if self
8047            .state
8048            .base_sync
8049            .as_ref()
8050            .is_some_and(|s| s.conflict.is_some())
8051            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8052                != Some(RunStatus::Gating)
8053            // `gate_ran == false` is not "passed" - `gate` leaves it false
8054            // both before it has ever run and when its last attempt was
8055            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8056            // permission to merge on nothing but the review record. Only a
8057            // gate that actually ran may proceed; an intentional local
8058            // review can also finish with verification explicitly not configured.
8059            // `RunState::gate_status` computes that distinction.
8060            || !self.state.gate_allows_progress()
8061        {
8062            return Ok(());
8063        }
8064        // This node's own record, not `status`: `status == Ready` is not
8065        // unique to the harmless `MergeMode::None` path this line was
8066        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8067        // run's PR was closed without merging — and on that run `mode` is
8068        // still `Pr`, so a reentry that fell through here would push and
8069        // open a second pull request. `self.state.merge` is set exactly once
8070        // this node (or `land`) has already produced a verdict, under every
8071        // mode, which is what "already done" actually means here.
8072        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8073            return Ok(());
8074        }
8075        let Some(winner) = self.state.winner().cloned() else {
8076            return Ok(());
8077        };
8078        let repo = self.state.repo.clone();
8079        let base = self.state.base_branch.clone();
8080        let mode = self.state.config.merge.mode;
8081        let style = self.state.config.merge.style;
8082        let facts = if is_review_run(&self.state) {
8083            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8084            let start = review_base(
8085                &repo,
8086                &self.state.config.merge.remote,
8087                &base,
8088                &self.state.base_commit,
8089                &winner.branch,
8090            )
8091            .await;
8092            branch_facts(&repo, &start, &winner.branch).await
8093        } else {
8094            None
8095        };
8096        // `None` when the base could not be freshly read: then an adopted
8097        // pull request's title is left alone.
8098        let leaked = if is_review_run(&self.state) {
8099            leaked_subjects(&self.state, &winner.branch).await
8100        } else {
8101            Some(Vec::new())
8102        };
8103        // `None`: parked on the owner's word about a withheld title or body.
8104        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8105        // the resume arrives here again.
8106        let Some(pr) = self
8107            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8108            .await?
8109        else {
8110            return Ok(());
8111        };
8112        let message = pr.commit_message();
8113
8114        let outcome = match mode {
8115            MergeMode::None => MergeOutcome {
8116                mode,
8117                ok: true,
8118                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8119                empty: false,
8120            },
8121            MergeMode::Pr | MergeMode::Local
8122                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8123            {
8124                MergeOutcome {
8125                    mode,
8126                    ok: false,
8127                    detail: empty_candidate_detail(&self.state, &base),
8128                    empty: true,
8129                }
8130            }
8131            MergeMode::Local => {
8132                let on = git::current_branch(&repo).await?;
8133                if on.as_deref() != Some(base.as_str()) {
8134                    MergeOutcome {
8135                        mode,
8136                        ok: false,
8137                        detail: format!(
8138                            "{} has {} checked out, not the base branch {base}",
8139                            repo.display(),
8140                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8141                        ),
8142                        empty: false,
8143                    }
8144                } else if !git::is_clean(&repo).await? {
8145                    MergeOutcome {
8146                        mode,
8147                        ok: false,
8148                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8149                        empty: false,
8150                    }
8151                } else {
8152                    let out = match style {
8153                        MergeStyle::Merge => {
8154                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8155                        }
8156                        MergeStyle::Squash => {
8157                            git::merge_squash(&repo, &winner.branch, &message).await?
8158                        }
8159                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8160                    };
8161                    MergeOutcome {
8162                        mode,
8163                        ok: out.ok(),
8164                        detail: if out.ok() { out.stdout } else { out.stderr },
8165                        empty: false,
8166                    }
8167                }
8168            }
8169            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8170                self.state.pr_publication_pending = true;
8171                self.state.save()?;
8172                let out =
8173                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8174                self.state.pr_publication_pending = out.is_err();
8175                MergeOutcome {
8176                    mode,
8177                    ok: out.is_ok(),
8178                    detail: match out {
8179                        Ok(url) => url,
8180                        Err(e) => format!("{e:#}"),
8181                    },
8182                    empty: false,
8183                }
8184            }
8185            MergeMode::Pr => {
8186                let remote = self.state.config.merge.remote.clone();
8187                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8188                if !pushed.ok() {
8189                    MergeOutcome {
8190                        mode,
8191                        ok: false,
8192                        detail: pushed.stderr,
8193                        empty: false,
8194                    }
8195                } else {
8196                    // A retry or resume of a run whose branch already has an
8197                    // open pull request adopts it rather than failing on a
8198                    // duplicate. Only this winner branch into this base:
8199                    // `branch_for` derives the name from the run id, so a
8200                    // different run's pull request never matches.
8201                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8202                    let out = match pr_merge_plan(found) {
8203                        PrPlan::Create => {
8204                            gh_pr_create(
8205                                &winner.worktree,
8206                                &base,
8207                                &winner.branch,
8208                                &pr.title,
8209                                &pr.body,
8210                            )
8211                            .await
8212                        }
8213                        PrPlan::Adopt { url, title } => {
8214                            self.state
8215                                .event("merge", format!("Pr: adopted open pull request {url}"));
8216                            if self.state.github_text.as_ref().is_some_and(|g| {
8217                                g.resolved
8218                                    && g.choice == Some(GithubTextChoice::Rewrite)
8219                                    && g.rewrite.is_some()
8220                            }) {
8221                                // The owner chose the entire pair, even if the
8222                                // existing title already matches or was edited.
8223                                land::set_pr_text_verbatim(
8224                                    &winner.worktree,
8225                                    &url,
8226                                    &pr.title,
8227                                    &pr.body,
8228                                )
8229                                .await
8230                                .map(|()| url)
8231                            } else {
8232                                if title != pr.title
8233                                    && (!is_review_run(&self.state)
8234                                        || leaked
8235                                            .as_deref()
8236                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8237                                    && let Err(e) =
8238                                        if self.state.github_text.as_ref().is_some_and(|g| {
8239                                            g.resolved
8240                                                && g.title
8241                                                && g.choice == Some(GithubTextChoice::Original)
8242                                        }) {
8243                                            // The owner chose to post this title unmodified.
8244                                            land::set_pr_title_verbatim(
8245                                                &winner.worktree,
8246                                                &url,
8247                                                &pr.title,
8248                                            )
8249                                            .await
8250                                        } else {
8251                                            land::set_pr_title(
8252                                                &mut self.state,
8253                                                &winner.worktree,
8254                                                &url,
8255                                                &pr.title,
8256                                            )
8257                                            .await
8258                                        }
8259                                {
8260                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8261                                    self.state
8262                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8263                                }
8264                                Ok(url)
8265                            }
8266                        }
8267                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8268                    };
8269                    match out {
8270                        Ok(url) => MergeOutcome {
8271                            mode,
8272                            ok: true,
8273                            detail: url,
8274                            empty: false,
8275                        },
8276                        Err(e) => MergeOutcome {
8277                            mode,
8278                            ok: false,
8279                            detail: e.to_string(),
8280                            empty: false,
8281                        },
8282                    }
8283                }
8284            }
8285        };
8286
8287        self.state.status = match (mode, outcome.ok) {
8288            (MergeMode::None, _) => RunStatus::Ready,
8289            (_, true) if outcome.empty => RunStatus::Ready,
8290            (MergeMode::Pr, true) => RunStatus::Ready,
8291            (_, true) => RunStatus::Merged,
8292            (_, false) => RunStatus::Blocked,
8293        };
8294        self.state.event(
8295            "merge",
8296            format!(
8297                "{:?}: {}",
8298                mode,
8299                outcome.detail.lines().next().unwrap_or("")
8300            ),
8301        );
8302        self.state.merge = Some(outcome);
8303        self.state.save()?;
8304
8305        // The PR is open and the run would historically stop here, leaving the
8306        // operator to watch checks, feed review comments back to a fixer, and
8307        // merge. That was done by hand six times in one session before this
8308        // existed. Opt-in, because merging is the one irreversible thing magi
8309        // can do to a repository.
8310        if self.state.config.graph.land
8311            && mode == MergeMode::Pr
8312            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8313        {
8314            self.run_land().await?;
8315        }
8316        // `run_land` may have left `status` at `Landing` - still waiting on
8317        // CI or the owner's approval, not actually settled - so this has to
8318        // read whatever `status` ended up as here, not the `Merged` this
8319        // function set a few lines up.
8320        self.settle_questions();
8321        Ok(())
8322    }
8323
8324    /// Enter `land`.
8325    ///
8326    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8327    /// a resumed run's re-entry. `land::land` itself is what serialises the
8328    /// two git-mutating moments inside the loop — the rebase push and
8329    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8330    /// to hold a lock across the whole call, and doing so would serialise
8331    /// this run's CI wait against a *different* run's land-approval resume
8332    /// in the same repository, which is exactly the "must not wait on
8333    /// another task" property the daemon's slot-freeing exists to give.
8334    async fn run_land(&mut self) -> Result<()> {
8335        crate::pr_head::ensure_owner(&self.state)?;
8336        let url = self
8337            .state
8338            .pr
8339            .as_ref()
8340            .map(|pr| pr.url.clone())
8341            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8342            .unwrap_or_default();
8343        let url = url.lines().next().unwrap_or("").trim().to_owned();
8344        if !url.starts_with("http") {
8345            return Ok(());
8346        }
8347        // A land failure is not a lost run: the work is on a branch and the
8348        // pull request is open, which is exactly where a human takes over.
8349        loop {
8350            match land::land(&mut self.state, &url).await {
8351                Ok(_)
8352                    if self
8353                        .state
8354                        .merge_repairs
8355                        .last()
8356                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8357                {
8358                    if self.repair_merge_review().await? {
8359                        continue;
8360                    }
8361                }
8362                Ok(pr) if self.state.parked => {
8363                    // `land` already saved the parked marker; nothing here
8364                    // overrides `status` back to a terminal value while an
8365                    // approval is still outstanding.
8366                    let _ = pr;
8367                }
8368                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8369                    self.state.status = RunStatus::Superseded;
8370                    self.state.save()?;
8371                }
8372                Ok(pr) => {
8373                    self.state.status = match pr.state {
8374                        land::PrLifecycle::Merged => RunStatus::Merged,
8375                        _ => RunStatus::Blocked,
8376                    };
8377                    // Downstream of a confirmed merge only - see
8378                    // `bump::should_release_bump`'s own doc for why this one
8379                    // check covers all three of `land`'s success paths.
8380                    // Best-effort: the run already landed, so a failure here
8381                    // (the decision call, `gh`, `cargo`) is recorded and never
8382                    // turns a landed run into a failed one.
8383                    if bump::should_release_bump(self.state.status)
8384                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8385                    {
8386                        // The event is the run's own record. Not-eligible cases
8387                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8388                        // `Err` is a bump that was tried and failed:
8389                        // `after_merge` itself raises the operator notice for
8390                        // that, whether or not a release PR exists yet.
8391                        self.state
8392                            .event("bump", format!("release bump skipped: {e:#}"));
8393                    }
8394                    // Independent of the bump, and best-effort in the same way:
8395                    // findings the merge left open become follow-up tasks.
8396                    if self.state.status == RunStatus::Merged {
8397                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8398                    }
8399                    self.state.save()?;
8400                }
8401                Err(e) => {
8402                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8403                        return Err(e);
8404                    }
8405                    self.state.status = RunStatus::Blocked;
8406                    self.state.event("land", format!("gave up: {e}"));
8407                    self.state.save()?;
8408                }
8409            }
8410            break;
8411        }
8412        Ok(())
8413    }
8414
8415    /// One repair cycle stays under the existing driver/task claim. An
8416    /// interrupted fixer is rolled back and held rather than run twice.
8417    async fn repair_merge_review(&mut self) -> Result<bool> {
8418        use crate::run::MergeRepairPhase as Phase;
8419        crate::pr_head::ensure_owner(&self.state)?;
8420        let i = self.state.merge_repairs.len() - 1;
8421        let request = self.state.merge_repairs[i].clone();
8422        if request.phase == Phase::Fixing {
8423            let restored = match self.state.winner().cloned() {
8424                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8425                None => Err(anyhow::anyhow!("repair has no winning branch")),
8426            };
8427            let why = match restored {
8428                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8429                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8430                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8431            };
8432            self.hold_merge_repair(i, why)?;
8433            return Ok(false);
8434        }
8435        let failure = if i >= self.state.config.graph.land_rounds
8436            || self.state.config.graph.review_rounds == 0
8437        {
8438            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8439        } else {
8440            None
8441        };
8442        if let Some(why) = failure {
8443            self.hold_merge_repair(i, why)?;
8444            return Ok(false);
8445        }
8446        if request.phase == Phase::Requested {
8447            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8448                Ok(pr) => pr,
8449                Err(e) => {
8450                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8451                    return Ok(false);
8452                }
8453            };
8454            self.state.merge_repairs[i].phase = Phase::Fixing;
8455            self.state.land_approval = None;
8456            self.state.land_armed_head = None;
8457            self.state.save()?;
8458            // Serialize the exact records, including finding ids/details,
8459            // reconsideration reasons, final votes and verification output.
8460            let evidence = serde_json::to_string_pretty(&request)?;
8461            let reason = format!(
8462                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8463            );
8464            let budget = self.state.config.graph.land_rounds;
8465            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8466            match result {
8467                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8468                Ok(land::Fixed::Failed(why)) => {
8469                    self.hold_merge_repair(
8470                        i,
8471                        format!(
8472                            "repair failed: {why}; inspect the branch and run an explicit review"
8473                        ),
8474                    )?;
8475                    return Ok(false);
8476                }
8477                Err(e) => {
8478                    self.hold_merge_repair(
8479                        i,
8480                        format!(
8481                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8482                        ),
8483                    )?;
8484                    return Ok(false);
8485                }
8486            }
8487            // Even no change gets a real new reviewer wave. The previous
8488            // evidence remains in the request; a former clean/concluded round
8489            // cannot short-circuit the new review or gate.
8490            self.begin_merge_repair_review(i)?;
8491        }
8492        self.review_loop().await?;
8493        self.gate().await?;
8494        if self.resource_halt()? || self.state.parked {
8495            return Ok(false);
8496        }
8497        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8498            != Some(RunStatus::Gating)
8499            || !self.state.gate_allows_progress()
8500        {
8501            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8502            return Ok(false);
8503        }
8504        // Review and gate may have added commits after the initial repair.
8505        // Publish exactly their final head to the existing PR, with a lease.
8506        let winner = self
8507            .state
8508            .winner()
8509            .cloned()
8510            .context("repair has no winning branch")?;
8511        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8512        if self.state.gate_head.as_deref() != Some(after.as_str())
8513            || !git::is_clean(&winner.worktree).await?
8514        {
8515            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8516            return Ok(false);
8517        }
8518        let expected = self
8519            .state
8520            .pr_pushed_head
8521            .as_deref()
8522            .unwrap_or(&request.head)
8523            .to_owned();
8524        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8525        // Push the checked commit itself: a local ref moving after the check
8526        // must not change which commit the forge receives.
8527        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8528        let refspec = format!("{after}:refs/heads/{public}");
8529        let push = git::git_raw(
8530            &winner.worktree,
8531            &["push", &lease, &self.state.config.merge.remote, &refspec],
8532        )
8533        .await;
8534        match push {
8535            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8536            Ok(out) => {
8537                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8538                return Ok(false);
8539            }
8540            Err(e) => {
8541                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8542                return Ok(false);
8543            }
8544        }
8545        self.state.merge_repairs[i].phase = Phase::Complete;
8546        self.state.status = RunStatus::Landing;
8547        self.state.event(
8548            "repair",
8549            "new review and gate completed; fresh owner merge decision required",
8550        );
8551        self.state.save()?;
8552        Ok(true)
8553    }
8554
8555    fn merge_repair_artifacts(&self) -> PathBuf {
8556        let artifacts = agent::artifacts_dir(&self.state.dir());
8557        match self
8558            .state
8559            .merge_repairs
8560            .last()
8561            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8562        {
8563            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8564            None => artifacts,
8565        }
8566    }
8567
8568    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8569        self.state.reviews.clear();
8570        self.state.gate.clear();
8571        self.state.gate_ran = false;
8572        self.state.gate_head = None;
8573        self.state.gate_fixes.clear();
8574        self.state.gate_repair_e2e = None;
8575        self.state.pre_gate.clear();
8576        self.state.pre_gate_commit = None;
8577        self.state.contested_handoff = None;
8578        self.state.land_reruns = None;
8579        self.state.land_approval = None;
8580        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8581        self.state.status = RunStatus::Reviewing;
8582        self.state.save()
8583    }
8584
8585    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8586        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8587        self.state.merge_repairs[index].failure = Some(why.clone());
8588        self.state.status = RunStatus::Blocked;
8589        self.state.event("repair", why);
8590        self.state.save()
8591    }
8592
8593    // -------------------------------------------------------------- helpers
8594
8595    /// Fetch or create a seat, keeping its conversation across nodes.
8596    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8597        if let Some(existing) = self.state.seats.get(key)
8598            && existing.agent == agent
8599        {
8600            return existing.clone();
8601        }
8602        // A seat that changes agent mints its session id from the agent too,
8603        // like a handover: the old agent's uuid is already taken by the CLI.
8604        let fresh = if self.state.seats.contains_key(key) {
8605            handover_seat(key, agent, self.state.next_seat_seed())
8606        } else {
8607            SeatState::new(key, agent, self.state.seed)
8608        };
8609        self.state.seats.insert(key.to_owned(), fresh.clone());
8610        fresh
8611    }
8612
8613    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8614    /// seat to another roster agent, in which case that agent. Nodes that
8615    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8616    /// reconsideration) must keep talking to whoever answered it, not slip
8617    /// back to the agent that failed it.
8618    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8619        match self.state.seats.get(key) {
8620            Some(s) if s.agent != spec.id => {
8621                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8622            }
8623            _ => spec,
8624        }
8625    }
8626
8627    /// A candidate rendered for judging, with the leak policy applied.
8628    fn view(&self, c: &Candidate) -> CandidateView {
8629        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8630            .unwrap_or_default();
8631        let (patch, _) = blind::sanitize_patch(
8632            &format!("candidate {} patch", c.label),
8633            &raw,
8634            &self.state.config.blind,
8635        );
8636        CandidateView {
8637            label: c.label,
8638            branch: c.branch.clone(),
8639            summary: c.summary.clone(),
8640            stat: c.stat.clone(),
8641            patch,
8642        }
8643    }
8644
8645    /// The full candidate set as prompt text, for seats with no live session.
8646    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8647        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8648        prompt::judge(
8649            "(see above)",
8650            &views,
8651            self.roles.judges.len(),
8652            base_short,
8653            "en",
8654        )
8655    }
8656
8657    /// The final-vote prompt with everything a seat that has no session of its
8658    /// own needs: the candidates, the seat's own ranking and reasons, and the
8659    /// anonymised deliberation it took part in (only when there was one, so a
8660    /// handed-over seat never sees more than the seat it replaces did). The
8661    /// `Final vote` heading stays first.
8662    fn vote_prompt_full(
8663        &self,
8664        j: usize,
8665        labels: &[char],
8666        language: &str,
8667        candidates: &[Candidate],
8668        base_short: &str,
8669    ) -> String {
8670        let mut text = format!(
8671            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8672            prompt::final_vote(labels, language),
8673            self.state.instruction,
8674            self.candidate_block(candidates, base_short)
8675        );
8676        if let Some(own) = self
8677            .state
8678            .judgements
8679            .get(j)
8680            .filter(|r| !r.ranking.is_empty())
8681        {
8682            let reasons = own
8683                .reasons
8684                .iter()
8685                .map(|(k, v)| format!("- {k}: {v}"))
8686                .collect::<Vec<_>>()
8687                .join("\n");
8688            text.push_str(&format!(
8689                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8690                own.ranking.iter().collect::<String>(),
8691                if reasons.is_empty() {
8692                    ""
8693                } else {
8694                    ", because:\n"
8695                }
8696            ));
8697        }
8698        if !self.state.deliberation.is_empty() {
8699            text.push_str("\n# What was argued before this vote\n");
8700            for t in self.transcript(&[], j) {
8701                text.push_str(&format!(
8702                    "\n## {}{}\n\n{}\n",
8703                    t.who,
8704                    if t.is_self { " (you)" } else { "" },
8705                    t.body.trim()
8706                ));
8707            }
8708        }
8709        text
8710    }
8711
8712    /// Anonymised transcript for judge `self_idx`.
8713    ///
8714    /// The initial rankings are always the opening statements. Seeding them
8715    /// only when no turn had been taken yet meant every judge after the first
8716    /// argued against a single voice instead of against the actual split — the
8717    /// disagreement is the information, so it is always on the table.
8718    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8719        let mut turns = Vec::new();
8720        for j in &self.state.judgements {
8721            if j.ranking.is_empty() {
8722                continue;
8723            }
8724            let reasons = j
8725                .reasons
8726                .iter()
8727                .map(|(k, v)| format!("- {k}: {v}"))
8728                .collect::<Vec<_>>()
8729                .join("\n");
8730            turns.push(Turn {
8731                who: format!("Judge {} (opening ranking)", j.judge),
8732                is_self: j.judge == self_idx + 1,
8733                body: format!(
8734                    "Ranked {}{}{reasons}",
8735                    j.ranking.iter().collect::<String>(),
8736                    if reasons.is_empty() {
8737                        ""
8738                    } else {
8739                        ", because:\n"
8740                    }
8741                ),
8742            });
8743        }
8744        for t in self
8745            .state
8746            .deliberation
8747            .iter()
8748            .flat_map(|r| r.turns.iter())
8749            .chain(current)
8750        {
8751            turns.push(Turn {
8752                who: format!("Judge {}", t.judge),
8753                is_self: t.judge == self_idx + 1,
8754                body: t.body.clone(),
8755            });
8756        }
8757        turns
8758    }
8759}
8760
8761/// Does this seat still hold the context a follow-up prompt would rely on?
8762fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8763    agent::has_session(spec.kind, seat, sessions)
8764}
8765
8766/// The next entry in `roster` after `start`, never wrapping back to the
8767/// front, whose id is not in `tried` yet.
8768///
8769/// Starts one past `start` rather than at the front of `roster`: `start` is
8770/// the seat's own original position, and a seat whose candidate slot already
8771/// sits on the roster's second entry must fall through to the third next, not
8772/// restart at the first — which is very likely a different candidate's own
8773/// agent already. Never wraps back past `start`, for the same reason: an
8774/// entry earlier in the roster than the seat's own position is almost
8775/// certainly some *other* candidate slot's own agent, and once the tail of
8776/// the roster is exhausted there are no more untried agents for *this* seat
8777/// to fall through to — the caller's fallback chain ends there, exactly as
8778/// "no further untried agents remain in the list for that seat" asks for.
8779///
8780/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8781/// the same id twice (an operator's `roles.implementers` typo, or a
8782/// `[[agents]]` list reused across roles) must not let
8783/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8784/// over `roster` either finds an untried id or runs out, so this always
8785/// terminates regardless of duplicates.
8786fn next_untried_in_roster<'a>(
8787    roster: &'a [AgentSpec],
8788    start: usize,
8789    tried: &BTreeSet<String>,
8790) -> Option<&'a AgentSpec> {
8791    roster
8792        .get(start + 1..)?
8793        .iter()
8794        .find(|s| !tried.contains(&s.id))
8795}
8796
8797/// The successor for a seat, shared by all four seat kinds (implement, judge,
8798/// review, advise). `others` holds the ids that *currently* occupy the other
8799/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
8800/// sees them), so roster entries beyond the seat count act as spares: a failed
8801/// seat goes to an agent no other seat holds whenever the roster permits.
8802///
8803/// `carried` is `Some` only for the review loop's carried failure history.
8804/// Order: (1) forward from `start`, never wrapping, untried, not a carried
8805/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
8806/// whole roster: untried and not another seat's occupant; (3) the plain walk
8807/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
8808/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
8809/// a better outcome than an empty seat, and it keeps the answer to "is there
8810/// a successor at all" exactly what it was before occupants were considered,
8811/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
8812/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
8813/// carried failure is a cheaper bet than doubling an agent on the panel, and
8814/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
8815/// the same round are handled one at a time, so a seat later in the batch
8816/// sees an earlier one's new occupant but not yet its own freed agent.
8817fn pick_successor<'a>(
8818    roster: &'a [AgentSpec],
8819    start: usize,
8820    tried: &BTreeSet<String>,
8821    carried: Option<&BTreeSet<String>>,
8822    others: &BTreeSet<String>,
8823) -> Option<&'a AgentSpec> {
8824    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8825    roster
8826        .get(start + 1..)
8827        .and_then(|tail| {
8828            tail.iter()
8829                .filter(free)
8830                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8831        })
8832        .or_else(|| {
8833            carried
8834                .is_some()
8835                .then(|| roster.iter().find(free))
8836                .flatten()
8837        })
8838        .or_else(|| match carried {
8839            Some(c) => next_for_seat(roster, start, tried, c),
8840            None => next_untried_in_roster(roster, start, tried),
8841        })
8842}
8843
8844/// The next agent for a seat that carries its failure history across rounds
8845/// (the review loop). `round_tried` is this round's own bound and starts
8846/// empty every round; `carried_failed` only decides priority.
8847///
8848/// First: an id walking forward from `start`, never wrapping, that is neither
8849/// tried this round nor failed in an earlier one. Only when that is exhausted
8850/// does it rescue: the first roster id (in roster order, so this one *does*
8851/// look before `start`) not yet tried this round, which is by then a carried
8852/// failure. Each id is rescued at most once per round, so it cannot loop.
8853fn next_for_seat<'a>(
8854    roster: &'a [AgentSpec],
8855    start: usize,
8856    round_tried: &BTreeSet<String>,
8857    carried_failed: &BTreeSet<String>,
8858) -> Option<&'a AgentSpec> {
8859    roster
8860        .get(start + 1..)?
8861        .iter()
8862        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8863        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8864}
8865
8866/// Where a reviewer seat starts a round: the agent that last answered it when
8867/// it is still on the roster and not marked failed, else the spec's own agent
8868/// unless it failed, else the next roster agent that has not failed, else the
8869/// spec's own agent again (the whole roster failed). Ids no longer on the
8870/// roster are ignored. An empty roster has no handover, so the spec stands.
8871fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8872    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8873        return spec;
8874    };
8875    let ok = |id: &str| !h.failed.contains(id);
8876    if let Some(last) = h.last_ok.as_deref()
8877        && ok(last)
8878        && let Some(s) = roster.iter().find(|s| s.id == last)
8879    {
8880        return s.clone();
8881    }
8882    if ok(&spec.id) {
8883        return spec;
8884    }
8885    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8886    roster
8887        .get(from + 1..)
8888        .into_iter()
8889        .flatten()
8890        .chain(roster.iter())
8891        .find(|s| ok(&s.id))
8892        .cloned()
8893        .unwrap_or(spec)
8894}
8895
8896/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
8897/// seed so a CLI that mints its session id up front (`--session-id`) never
8898/// reuses the uuid the previous agent already opened under the same seat key.
8899pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8900    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8901}
8902
8903/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
8904/// so the classifier below and the code that builds the message cannot drift.
8905const TIMED_OUT: &str = "timed out";
8906
8907impl FailClass {
8908    /// `None` for an answer; otherwise how the turn failed.
8909    fn of(out: &AgentOutcome) -> Option<Self> {
8910        match out {
8911            AgentOutcome::Ok(_) => None,
8912            AgentOutcome::Quota(_) => Some(Self::Quota),
8913            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8914            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8915            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8916            // Not a failure of the agent. Callers check for it first; `None`
8917            // here means "no class to hand over on".
8918            AgentOutcome::ResourceBlocked(_) => None,
8919        }
8920    }
8921
8922    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
8923    fn stem_word(&self) -> &'static str {
8924        match self {
8925            Self::Quota => "quota",
8926            _ => "handover",
8927        }
8928    }
8929}
8930
8931/// The message's first line with its variable parts removed — digit runs and
8932/// path-like tokens — so "exit code 2" and "exit code 7" read
8933/// as one kind of failure.
8934fn failure_signature(msg: &str) -> String {
8935    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8936    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
8937    // the historical machine key while the process diagnosis uses readable prose.
8938    if let Some(code) = line
8939        .strip_prefix("process exited with no usable output (exit code ")
8940        .and_then(|s| s.strip_suffix(')'))
8941        .and_then(|s| s.parse::<i32>().ok())
8942    {
8943        line = format!("exited with some({code}) and no usable output");
8944    } else if matches!(
8945        line.as_str(),
8946        "process exited with no usable output (no exit code reported)"
8947            | "process exited with no usable output (exit code unavailable)"
8948    ) {
8949        line = "exited with none and no usable output".to_owned();
8950    }
8951    let mut out = Vec::new();
8952    for word in line.split_whitespace() {
8953        if word.contains('/') || word.contains('\\') {
8954            out.push("<path>".to_owned());
8955            continue;
8956        }
8957        let mut w = String::new();
8958        let mut in_digits = false;
8959        for c in word.chars() {
8960            if c.is_ascii_digit() {
8961                if !in_digits {
8962                    w.push('#');
8963                }
8964                in_digits = true;
8965            } else {
8966                in_digits = false;
8967                w.push(c);
8968            }
8969        }
8970        out.push(w);
8971    }
8972    out.join(" ").chars().take(120).collect()
8973}
8974
8975/// Whether a seat that just failed with `cur` may go to the next roster agent.
8976/// A quota or a timeout always may. Any other failure may not when the agent
8977/// before it failed the same way: an error the prompt causes would otherwise
8978/// walk the whole roster. `prev` is the class of the immediately preceding
8979/// agent's failure, so a quota or timeout in between breaks the run of
8980/// identical failures by itself.
8981fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8982    match cur {
8983        FailClass::Quota | FailClass::Timeout => true,
8984        FailClass::Other(_) => prev != Some(cur),
8985    }
8986}
8987
8988/// A short human reason for a failed outcome, for the handover record.
8989fn fail_reason(out: &AgentOutcome) -> String {
8990    match out {
8991        AgentOutcome::Ok(_) => String::new(),
8992        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8993        AgentOutcome::Dropped(o) => format!(
8994            "the CLI dropped the stream ({})",
8995            o.dropped
8996                .as_ref()
8997                .map(|d| d.why.as_str())
8998                .unwrap_or("it ended without delivering its answer")
8999        ),
9000        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9001            e.lines().next().unwrap_or("").chars().take(160).collect()
9002        }
9003    }
9004}
9005
9006/// Note one handover in the run: the structured record and, in the timeline,
9007/// the sentence a person reads. A quota keeps the wording it always had.
9008pub(crate) fn record_handover(
9009    state: &mut RunState,
9010    node: &str,
9011    seat: &str,
9012    from: &str,
9013    to: &str,
9014    class: &FailClass,
9015    reason: &str,
9016) {
9017    let message = if *class == FailClass::Quota {
9018        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9019    } else {
9020        format!("{seat}: handed over {from} -> {to} ({reason})")
9021    };
9022    state.event(node, message);
9023    state.handovers.push(Handover {
9024        at: Timestamp::now(),
9025        node: node.to_owned(),
9026        seat: seat.to_owned(),
9027        from: from.to_owned(),
9028        to: to.to_owned(),
9029        reason: reason.to_owned(),
9030    });
9031}
9032
9033/// Did this reply report running a command whose own CLI never confirmed an
9034/// exit status?
9035///
9036/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9037/// command *finished* (see that type's own doc), so this can only be `true`
9038/// for a command whose completion event carried no readable exit code — not
9039/// for one that simply is not mentioned at all. That is the one signal this
9040/// crate can read, from the same record `state.jobs` renders, about a reply
9041/// standing next to work its own CLI cannot vouch for finishing; it is
9042/// deliberately not a check on the exit code's *value* (a fixer legitimately
9043/// runs a command that fails mid-iteration before it succeeds) and not a
9044/// guess at a command still running in the background (which emits no event
9045/// at all, and so leaves no evidence here to find).
9046fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9047    commands.iter().any(|c| c.exit_code.is_none())
9048}
9049
9050/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9051/// trusted as a verified no-op — the adoption guard's own text-level half.
9052///
9053/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9054/// timed out): a marker only earns the benefit of the doubt from a turn the
9055/// CLI itself vouches for finishing properly, the same house style
9056/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9057/// *fix* report to for `commands`. A candidate that timed out, exited
9058/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9059/// is, whatever prose it wrote — this returns `None` before it ever looks at
9060/// `text`. The remaining guards (the tree really is empty, the evidence is
9061/// non-empty) are the caller's: this only reads what the reply *claimed*.
9062fn verified_noop_claim(
9063    usable: bool,
9064    commands: &[agent::CommandEvidence],
9065    text: &str,
9066) -> Option<String> {
9067    (usable && !has_unconfirmed_command(commands))
9068        .then(|| verdict::verified_noop(text))
9069        .flatten()
9070}
9071
9072fn short(commit: &str) -> String {
9073    commit.chars().take(7).collect()
9074}
9075
9076fn make_executable(path: &Path) -> Result<()> {
9077    #[cfg(unix)]
9078    {
9079        use std::os::unix::fs::PermissionsExt as _;
9080        let mut perms = std::fs::metadata(path)?.permissions();
9081        perms.set_mode(0o755);
9082        std::fs::set_permissions(path, perms)?;
9083    }
9084    #[cfg(not(unix))]
9085    {
9086        let _ = path;
9087    }
9088    Ok(())
9089}
9090
9091/// What every seat in one batch shares: where the answers are attributed, the
9092/// prompt overlay they inherit, and the build cache they are told to use.
9093///
9094/// A struct rather than four more parameters: `wave` also needs the run's
9095/// state (to record who is answering right now) and the attempt number, and
9096/// eight positional arguments is both unreadable and a clippy error.
9097struct WaveCtx<'a> {
9098    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9099    /// paid for it.
9100    run: &'a str,
9101    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9102    node: &'a str,
9103    prompts: &'a Prompts,
9104    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9105    cache: Option<&'a Path>,
9106    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9107    /// `JobRecord::round`. `None` for every other node.
9108    round: Option<usize>,
9109    /// Carry each seat's failed-agent history across waves (the review loop
9110    /// only): start-of-round priority and handover choice read
9111    /// [`RunState::seat_history`], and every answer or failure writes it.
9112    carry_seats: bool,
9113    /// The run's park / interrupt requests, so a wait for a shared resource
9114    /// can be abandoned the moment the operator asks to park.
9115    pauses: Option<(&'a Pause, &'a Pause)>,
9116}
9117
9118/// Shared scheduling policy for operations and spawned seat invocations.
9119fn repository_wait(
9120    state: &RunState,
9121    pauses: Option<(&Pause, &Pause)>,
9122) -> crate::repo_lock::WaitPolicy {
9123    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9124    let inherited = crate::repo_lock::WaitPolicy::current();
9125    crate::repo_lock::WaitPolicy::new(
9126        Duration::from_secs(state.config.graph.timeout_resource_wait),
9127        move || {
9128            inherited.cancelled()
9129                || pauses
9130                    .as_ref()
9131                    .is_some_and(|(a, b)| a.parked() || b.parked())
9132        },
9133    )
9134}
9135
9136fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9137    if error.is::<crate::repo_lock::ResourceBlocked>() {
9138        state.resource_block = Some(crate::run::ResourceBlock {
9139            node: "repository".into(),
9140            kind: if error.to_string().contains("disk pressure") {
9141                "disk pressure"
9142            } else {
9143                "repository coordination"
9144            }
9145            .into(),
9146            detail: error.to_string(),
9147            at: Timestamp::now(),
9148        });
9149        state.event("resource", error.to_string());
9150    }
9151}
9152
9153async fn coordinate_operation(
9154    state: &mut RunState,
9155    cwd: &Path,
9156    wait: &crate::repo_lock::WaitPolicy,
9157) -> Result<Option<crate::repo_lock::SeatLease>> {
9158    let result =
9159        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9160            .await;
9161    if let Err(e) = &result {
9162        record_repository_block(state, e);
9163    }
9164    result
9165}
9166
9167/// Run one job, honouring the parallelism budget.
9168async fn run_one(
9169    job: SeatJob,
9170    sem: Arc<Semaphore>,
9171    ctx: &WaveCtx<'_>,
9172    state: &mut RunState,
9173    attempt: usize,
9174) -> (SeatState, AgentOutcome) {
9175    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9176        .await
9177        .pop()
9178        .expect("one job in, one result out");
9179    (seat, out)
9180}
9181
9182/// Run every job concurrently, capped by the semaphore, preserving order.
9183///
9184/// Every seat in the batch is recorded into [`RunState::active`] before the
9185/// wave starts and cleared as each answer lands, so the run's own record says
9186/// who is still being waited on rather than only who finished.
9187async fn wave(
9188    jobs: Vec<SeatJob>,
9189    sem: Arc<Semaphore>,
9190    ctx: &WaveCtx<'_>,
9191    state: &mut RunState,
9192    attempt: usize,
9193) -> Vec<(usize, SeatState, AgentOutcome)> {
9194    let WaveCtx {
9195        run,
9196        node,
9197        prompts,
9198        cache,
9199        round,
9200        carry_seats: _,
9201        pauses,
9202    } = *ctx;
9203    for job in &jobs {
9204        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9205    }
9206    if let Err(e) = state.save() {
9207        // A failed persist of "who is answering right now" must not abort the
9208        // wave: the seats are already being asked, and the alternative is
9209        // losing the answers to save a status line nobody may even be
9210        // watching.
9211        tracing::warn!("could not persist in-progress seats: {e:#}");
9212    }
9213    // Hold the shared build cache's lease for the whole batch, not per job:
9214    // several candidates (an implement wave) or a fixer legitimately share
9215    // one cache concurrently within this run, and that stays untouched — a
9216    // single lease taken once for the whole wave and released once it is
9217    // done is what stops a *different* borrower (another run's own wave, its
9218    // e2e/gate, a human's `magi review`) from interleaving a build into the
9219    // same directory while this one is in flight. Best-effort, not
9220    // all-or-nothing: a wave that cannot get the lease within its own
9221    // longest job's budget still runs — an hour of paid implementer calls is
9222    // not thrown away over cache contention — but every write-allowed seat
9223    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9224    // below), the same fallback a read-only seat always gets, rather than
9225    // building into a directory this run was never granted. The identity
9226    // record is still invalidated below either way, so the next tracked
9227    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9228    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9229    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9230    // never borrowed from a job's agent budget (and never charged to it either,
9231    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9232    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9233    let cache_guard = if let Some(cache_dir) = cache {
9234        if jobs_had_a_writer {
9235            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9236            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9237            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9238            {
9239                Ok(Some(g)) => Some(g),
9240                // Parking: leave without spawning anything. The run parks at
9241                // the next boundary and the seats are asked afresh on resume.
9242                Ok(None) => {
9243                    state.resource_block = Some(crate::run::ResourceBlock {
9244                        node: node.to_owned(),
9245                        kind: "shared build cache".to_owned(),
9246                        detail: "the run is parking".to_owned(),
9247                        at: Timestamp::now(),
9248                    });
9249                    return block_wave(
9250                        jobs,
9251                        state,
9252                        node,
9253                        attempt,
9254                        "parked while waiting for the shared build cache",
9255                    );
9256                }
9257                // An unreadable lease is not contention: keep the old
9258                // fallback (no shared cache for this wave's seats).
9259                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9260                Err(e) => {
9261                    // Writing seats may not build into a cache this run does
9262                    // not hold, and an agent that cannot be given one is not
9263                    // asked at all: the run stops resumably instead.
9264                    let why = format!("{e:#}");
9265                    state.resource_block = Some(crate::run::ResourceBlock {
9266                        node: node.to_owned(),
9267                        kind: "shared build cache".to_owned(),
9268                        detail: why.clone(),
9269                        at: Timestamp::now(),
9270                    });
9271                    state.event(
9272                        "resource",
9273                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9274                    );
9275                    return block_wave(jobs, state, node, attempt, &why);
9276                }
9277            }
9278        } else {
9279            None
9280        }
9281    } else {
9282        None
9283    };
9284    // Every job keeps its whole budget: the wait above was not the agent's.
9285    // Restart the seats' clocks so "started" means "being spawned now".
9286    state.seats_spawning(node, attempt);
9287    if let Err(e) = state.save() {
9288        tracing::warn!("could not persist the start of a wave: {e:#}");
9289    }
9290    // Queueing behind this wave's own seats for a parallelism slot is normal
9291    // (each of them is bounded by its own agent timeout); only a wait beyond
9292    // all of them plus the resource limit means the slots never came back.
9293    let slot_limit = resource_limit
9294        + jobs
9295            .iter()
9296            .map(|j| j.timeout)
9297            .max()
9298            .unwrap_or_default()
9299            .saturating_mul(jobs.len() as u32);
9300    let mut set = tokio::task::JoinSet::new();
9301    // Seats announce themselves here the moment they hold a parallelism
9302    // permit, i.e. just before their CLI is spawned.
9303    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9304    let overlay = prompts.overlay(node);
9305    let repository_wait = repository_wait(state, pauses);
9306    for (i, mut job) in jobs.into_iter().enumerate() {
9307        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9308        if cache.is_some() {
9309            job.prompt.push('\n');
9310            job.prompt
9311                .push_str(&prompt::build_cache_note(node, job.allow_write));
9312        }
9313        let sem = Arc::clone(&sem);
9314        let run = run.to_owned();
9315        let node = node.to_owned();
9316        // Only implementers were told about the task's attachments, so only
9317        // their seats get the directory widened for reading.
9318        let attachments = if node == "implement" {
9319            state.attachments.clone()
9320        } else {
9321            Vec::new()
9322        };
9323        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9324        // `prompt::build_cache_note`'s doc for why setting it anyway is
9325        // exactly how a sandboxed reviewer's write refusal got reported as a
9326        // defect in the patch, not a property of its own seat. And a
9327        // write-allowed one is handed it only when the lease above was
9328        // actually acquired: a wave that could not get it (`cache_guard` is
9329        // `None`, see its own comment) must not send seats to build into a
9330        // directory this run does not hold - that is the exact concurrent,
9331        // unmanaged-write race this module exists to prevent, not something
9332        // "proceeding anyway" is allowed to reintroduce.
9333        let cache = cache
9334            .filter(|_| job.allow_write && cache_guard.is_some())
9335            .map(Path::to_path_buf);
9336        let spawned_tx = spawned_tx.clone();
9337        let repository_wait = repository_wait.clone();
9338        let coordination = state.config.coordination.clone();
9339        let fetched_base = state.fetched_base.clone();
9340        set.spawn(async move {
9341            // A wait for a parallelism slot is scheduling, not the agent: it
9342            // is bounded by its own limit and ends as a resource block, never
9343            // as a timeout of the agent that was not started.
9344            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9345                Ok(p) => p,
9346                Err(_) => {
9347                    return (
9348                        i,
9349                        job.seat,
9350                        AgentOutcome::ResourceBlocked(format!(
9351                            "no parallel slot became free within {}s",
9352                            slot_limit.as_secs()
9353                        )),
9354                        false,
9355                    );
9356                }
9357            };
9358            spawned_tx.send(job.seat.key.clone()).ok();
9359            let mut seat = job.seat;
9360            let out = repository_wait
9361                .scope(agent::invoke(
9362                    &job.spec,
9363                    &mut seat,
9364                    &Invocation {
9365                        scope: crate::agent::InvocationScope::Repository,
9366                        coordination: coordination.as_ref(),
9367                        fetched_base: fetched_base.as_ref(),
9368                        cwd: &job.cwd,
9369                        prompt: &job.prompt,
9370                        timeout: job.timeout,
9371                        allow_write: job.allow_write,
9372                        unsandboxed: false,
9373                        sessions: job.sessions,
9374                        artifacts: &job.artifacts,
9375                        stem: &job.stem,
9376                        run: &run,
9377                        node: &node,
9378                        cache_dir: cache.as_deref(),
9379                        attachments: &attachments,
9380                        writable: &[],
9381                    },
9382                ))
9383                .await;
9384            let interrupted = out
9385                .as_ref()
9386                .err()
9387                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9388            let out = match out {
9389                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9390                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9391                // Billed work the CLI failed to hand over is not an ordinary
9392                // failure, but its text is the CLI's raw error JSON, not an
9393                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9394                // read it as one by forgetting to check. `usable()` is always
9395                // false here (dropped implies an empty response), so this has
9396                // to be checked before the catch-all `Failed` below or the
9397                // one shape this exists for is lost with the rest.
9398                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9399                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9400                Ok(o) => AgentOutcome::Failed(format!(
9401                    "process exited with no usable output ({})",
9402                    crate::notices::exit_status(o.exit_code)
9403                )),
9404                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9405                    AgentOutcome::ResourceBlocked(e.to_string())
9406                }
9407                Err(e) => AgentOutcome::Failed(e.to_string()),
9408            };
9409            (i, seat, out, interrupted)
9410        });
9411    }
9412    drop(spawned_tx);
9413    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9414    loop {
9415        let joined = tokio::select! {
9416            Some(key) = spawned_rx.recv() => {
9417                // The clock of this seat starts now, not when it queued.
9418                state.seat_spawned(&key);
9419                if let Err(e) = state.save() {
9420                    tracing::warn!("could not persist a seat's start: {e:#}");
9421                }
9422                continue;
9423            }
9424            joined = set.join_next() => joined,
9425        };
9426        let Some(joined) = joined else {
9427            break;
9428        };
9429        let (i, seat, out, interrupted) = match joined {
9430            Ok(v) => v,
9431            // No seat to clear: a panicked task never reported which one it
9432            // was. The defensive sweep below this loop is what stops that
9433            // seat's `active` entry from surviving forever.
9434            Err(e) => {
9435                tracing::error!("agent task panicked: {e}");
9436                continue;
9437            }
9438        };
9439        state.seat_finished(&seat.key);
9440        if interrupted
9441            && node == "implement"
9442            && let Some(label) = state
9443                .candidates
9444                .iter()
9445                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9446                .map(|candidate| candidate.label)
9447        {
9448            state.candidate_preservation.insert(
9449                label,
9450                "interrupted implementation awaiting persistence".into(),
9451            );
9452            state.interrupted_implementations.insert(label);
9453        }
9454        if let AgentOutcome::ResourceBlocked(why) = &out
9455            && state.resource_block.is_none()
9456        {
9457            state.resource_block = Some(crate::run::ResourceBlock {
9458                node: node.to_owned(),
9459                kind: if why.contains("disk pressure") {
9460                    "disk pressure"
9461                } else if why.contains("repository coordination") {
9462                    "repository coordination"
9463                } else {
9464                    "parallel slot"
9465                }
9466                .to_owned(),
9467                detail: why.clone(),
9468                at: Timestamp::now(),
9469            });
9470            let activity = if interrupted {
9471                "agent execution interrupted"
9472            } else {
9473                "no agent was started"
9474            };
9475            state.event("resource", format!("{node}: {activity} - {why}"));
9476        }
9477        record_jobs(state, node, round, &seat.key, &out);
9478        if let Err(e) = state.save() {
9479            tracing::warn!("could not persist a seat's completion: {e:#}");
9480        }
9481        if collected.len() <= i {
9482            collected.resize_with(i + 1, || None);
9483        }
9484        collected[i] = Some((i, seat, out));
9485    }
9486    // Belt-and-braces for the panic branch above: every seat this exact batch
9487    // started shares this `(node, attempt)` pair, and every seat that finished
9488    // normally already cleared itself, so anything left tagged with it here
9489    // can only be a panicked task's leftover. Cleared unconditionally rather
9490    // than left to read as still answering forever.
9491    if state
9492        .active
9493        .values()
9494        .any(|a| a.node == node && a.attempt == attempt)
9495    {
9496        state
9497            .active
9498            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9499        if let Err(e) = state.save() {
9500            tracing::warn!("could not persist the end of a wave: {e:#}");
9501        }
9502    }
9503    // Whether or not the lease above was actually held, several worktrees
9504    // may just have built into the cache with nothing here able to name one
9505    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9506    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9507    // clean it might not have strictly needed; trusting a stale match would
9508    // cost it a wrong answer.
9509    if let Some(cache_dir) = cache
9510        && jobs_had_a_writer
9511    {
9512        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9513    }
9514    if let Some(guard) = cache_guard {
9515        guard.release();
9516    }
9517    collected.into_iter().flatten().collect()
9518}
9519
9520/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9521/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9522/// this is data collection, not the fix-specific completion contract in
9523/// [`Runner::continue_fix_report`], and applies regardless of which node
9524/// asked.
9525///
9526/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9527/// evidence from; `Failed` does not, and correctly contributes nothing — a
9528/// timeout or crash is not itself evidence about a command the seat may have
9529/// started.
9530fn record_jobs(
9531    state: &mut RunState,
9532    node: &str,
9533    round: Option<usize>,
9534    seat: &str,
9535    out: &AgentOutcome,
9536) {
9537    let commands: &[agent::CommandEvidence] = match out {
9538        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9539        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9540    };
9541    let checked_at = Timestamp::now();
9542    for c in commands {
9543        state.jobs.push(JobRecord {
9544            node: node.to_owned(),
9545            round,
9546            seat: seat.to_owned(),
9547            id: c.id.clone(),
9548            description: c.description.clone(),
9549            checked_at,
9550            status: match c.exit_code {
9551                Some(0) => JobStatus::Completed,
9552                Some(_) => JobStatus::Failed,
9553                None => JobStatus::Unknown,
9554            },
9555            exit_code: c.exit_code,
9556            result_summary: c.result_summary.clone(),
9557            source: c.source.clone(),
9558        });
9559    }
9560}
9561
9562/// Is a review round clean, given how many reviewer seats answered against
9563/// how many the round expected?
9564///
9565/// A seat that never answered (timeout, crash, unparsable output) is not a
9566/// seat that read the patch and found nothing — treating it as such is
9567/// exactly the bug this function exists to close. Under the default `block`
9568/// policy a missing seat can never be clean; `warn` still requires the seats
9569/// that *did* answer to have found nothing blocking and verification to be
9570/// green.
9571///
9572/// `quota_missing` narrows that `block` default for exactly one cause of
9573/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9574/// a session limit lifts by the very next round buys nothing — the seat is
9575/// asked again with the same quota — so once every missing seat is accounted
9576/// for by a quota loss (and at least one seat *did* answer, so a decision has
9577/// something to rest on) the round is decided on the panel that could answer,
9578/// same as `warn` would. A panel that lost every seat to quota is not
9579/// decided here: `answered == 0` falls through to the existing `block`
9580/// fallback so a fully collapsed panel still waits rather than landing on no
9581/// review at all.
9582fn round_is_clean(
9583    blocking: usize,
9584    e2e_ok: bool,
9585    answered: usize,
9586    expected: usize,
9587    quota_missing: usize,
9588    policy: IncompleteReviewPolicy,
9589) -> bool {
9590    if blocking != 0 || !e2e_ok {
9591        return false;
9592    }
9593    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9594        return true;
9595    }
9596    answered > 0 && expected - answered <= quota_missing
9597}
9598
9599/// The review loop's own conclusion, derived entirely from its persisted
9600/// round records and the round budget that produced them — never from
9601/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9602/// recomputes the identical answer regardless of what an earlier node in the
9603/// same walk, or a previous walk, did to `status`.
9604///
9605/// `None` while more rounds remain to try, including when review never ran
9606/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9607/// gone clean, or the budget is spent, or the tree has stopped moving (see
9608/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9609///
9610/// - An incomplete panel that raised nothing is missing input, not a
9611///   verified tree — never a hand-off candidate, whatever verification said
9612///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9613/// - Otherwise, green e2e on the last round hands off (see
9614///   [`Runner::stop_reviewing`]); red e2e blocks.
9615///
9616/// A last round whose own verification is still `ResourceBlocked` — magi
9617/// itself never got a command to run, not evidence the patch is broken —
9618/// is neither: this returns `None` for it too, the same as "more rounds
9619/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9620/// handling of that shape) instead of this cheap recomputation guessing a
9621/// verdict a real attempt never produced.
9622pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9623    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9624        return Some(RunStatus::Gating);
9625    }
9626    let last = reviews.last()?;
9627    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9628    if reviews.len() < max_rounds && !stagnant {
9629        return None;
9630    }
9631    if last.incomplete() && last.blocking == 0 {
9632        return Some(RunStatus::Blocked);
9633    }
9634    if last.e2e_status() == E2eStatus::ResourceBlocked {
9635        return None;
9636    }
9637    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9638        RunStatus::Gating
9639    } else {
9640        RunStatus::Blocked
9641    })
9642}
9643
9644/// How long a re-ask may take, given the budget the first attempt had.
9645///
9646/// A `nudged` retry is a request to restate an answer the seat has already
9647/// worked out: it carries no new work, so it does not deserve the original
9648/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9649/// 133 seconds while a third sat for over ten minutes on a resumed session
9650/// holding 410 KB of prior output - and because the retry had inherited the
9651/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9652/// judging round whose other seats were long finished.
9653///
9654/// A quarter of the budget, with a floor so that a deliberately short timeout
9655/// does not collapse to nothing. A retry that re-sends the whole prompt
9656/// (because the seat kept no context) is the original job again, and keeps the
9657/// original budget.
9658pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9659    if nudged {
9660        (full / 4).max(Duration::from_secs(120)).min(full)
9661    } else {
9662        full
9663    }
9664}
9665
9666/// Run a wave and parse each reply, re-asking the seats whose reply was
9667/// unusable.
9668///
9669/// The re-ask is a nudge rather than the whole prompt again when the seat still
9670/// holds its conversation, which is the difference between a cheap retry and
9671/// paying for the entire candidate set twice.
9672///
9673/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9674/// successor in `roster` is handed to it instead of being re-asked: the
9675/// handover is the retry. A seat with no successor left (a single-agent
9676/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9677/// the asks to one seat in one node number at most
9678/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9679/// is asked once (a dropped stream is nudged first), and only the last agent
9680/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9681/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9682/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9683/// other failure stops the chain when the previous agent failed the same way
9684/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9685/// [`has_context`] is false and the job's own full prompt and full budget are
9686/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9687/// intermediate ones are not losses) and is returned as a failure like any
9688/// other absent seat — the caller decides whether the panel still has a
9689/// quorum. An empty `roster` disables handover: failures are nudged as they
9690/// always were, and a quota is simply lost. A reply that fails to parse or
9691/// validate is the prompt's doing and is only ever nudged, never handed over.
9692///
9693/// The returned [`SeatState`] names the agent that answered (or tried last).
9694#[allow(clippy::too_many_arguments)]
9695async fn ask_json_wave<T>(
9696    jobs: Vec<SeatJob>,
9697    sem: Arc<Semaphore>,
9698    retries: usize,
9699    roster: &[AgentSpec],
9700    ctx: &WaveCtx<'_>,
9701    losses: &mut Vec<QuotaLoss>,
9702    state: &mut RunState,
9703    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9704) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9705where
9706    T: serde::de::DeserializeOwned + Send + 'static,
9707{
9708    ask_wave_with(
9709        jobs,
9710        sem,
9711        retries,
9712        roster,
9713        ctx,
9714        losses,
9715        state,
9716        &|text: &str| {
9717            let v = verdict::extract_json::<T>(text)?;
9718            validate(&v)?;
9719            Ok(v)
9720        },
9721    )
9722    .await
9723}
9724
9725/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9726/// a node whose answer is prose (deliberation) shares the same handover,
9727/// failure classification, quota bookkeeping and bounds instead of a copy.
9728///
9729/// A seat handed to another roster agent is sent the job's `handover` prompt
9730/// (when it has one) rather than `prompt`: the new agent has no session, so a
9731/// resume-style prompt would be incomplete. That holds for the handover ask
9732/// and for every nudge to that agent whose `has_context` is false.
9733#[allow(clippy::too_many_arguments)]
9734async fn ask_wave_with<T>(
9735    jobs: Vec<SeatJob>,
9736    sem: Arc<Semaphore>,
9737    retries: usize,
9738    roster: &[AgentSpec],
9739    ctx: &WaveCtx<'_>,
9740    losses: &mut Vec<QuotaLoss>,
9741    state: &mut RunState,
9742    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9743) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9744where
9745    T: Send + 'static,
9746{
9747    let n = jobs.len();
9748    let originals: Vec<SeatJob> = jobs;
9749    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9750    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9751    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9752    // answer, N once it has gone through N nudges. Read back once this
9753    // returns, so a caller building a history record (`ReviewRecord`) can
9754    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9755    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9756    // field's own doc.
9757    let mut nudges: Vec<usize> = vec![0; n];
9758    // The agent now occupying each seat, the ids it has already been through,
9759    // where in the roster the walk began, the class of the last failure, and
9760    // the stem word of a handover not yet asked (full prompt, full budget).
9761    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9762    let mut tried: Vec<BTreeSet<String>> = specs
9763        .iter()
9764        .map(|s| BTreeSet::from([s.id.clone()]))
9765        .collect();
9766    let starts: Vec<usize> = specs
9767        .iter()
9768        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9769        .collect();
9770    let carry = ctx.carry_seats && !roster.is_empty();
9771    // Carried across rounds: ids that failed the seat earlier, and how the
9772    // last failure went (so a repeat of it is not handed over again).
9773    let carried: Vec<BTreeSet<String>> = originals
9774        .iter()
9775        .map(|j| {
9776            state
9777                .seat_history
9778                .get(&j.seat.key)
9779                .filter(|_| carry)
9780                .map(|h| h.failed.clone())
9781                .unwrap_or_default()
9782        })
9783        .collect();
9784    let mut prev: Vec<Option<FailClass>> = originals
9785        .iter()
9786        .map(|j| {
9787            state
9788                .seat_history
9789                .get(&j.seat.key)
9790                .filter(|_| carry)
9791                .and_then(|h| h.last_fail.clone())
9792        })
9793        .collect();
9794    let next_agent =
9795        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9796            let others: BTreeSet<String> = specs
9797                .iter()
9798                .enumerate()
9799                .filter(|(j, _)| *j != i)
9800                .map(|(_, s)| s.id.clone())
9801                .collect();
9802            pick_successor(
9803                roster,
9804                starts[i],
9805                tried,
9806                carry.then(|| &carried[i]),
9807                &others,
9808            )
9809            .cloned()
9810        };
9811    let mut fresh: Vec<Option<String>> = vec![None; n];
9812    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9813    let mut pending: Vec<usize> = (0..n).collect();
9814
9815    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
9816    // asks: an agent with a successor is asked once and handed over, and only
9817    // a seat with no successor spends `retries` nudges on the same agent. This
9818    // only guarantees the loop's own termination whatever those say.
9819    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9820    for round in 0..max_rounds {
9821        if pending.is_empty() {
9822            break;
9823        }
9824        let mut batch = Vec::with_capacity(pending.len());
9825        let mut renudged: Vec<&str> = Vec::new();
9826        for &i in &pending {
9827            let src = &originals[i];
9828            // A seat now held by another agent than the job named has no
9829            // session of its own: it gets the full-context prompt whenever
9830            // it is asked in full (the handover ask, a nudge it cannot
9831            // resume).
9832            let full: &str = match &src.handover {
9833                Some(h) if specs[i].id != src.spec.id => h,
9834                _ => &src.prompt,
9835            };
9836            // The prompt and the budget are one decision: a nudge restates
9837            // finished work, a re-sent prompt redoes it.
9838            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9839                (
9840                    full.to_owned(),
9841                    src.timeout,
9842                    format!("{}-{word}-{}", src.stem, specs[i].id),
9843                )
9844            } else if nudges[i] == 0 {
9845                (full.to_owned(), src.timeout, src.stem.clone())
9846            } else {
9847                renudged.push(src.seat.key.as_str());
9848                let why = done[i]
9849                    .as_ref()
9850                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
9851                    .unwrap_or_else(|| "no parsable answer".to_owned());
9852                let nudge = prompt::nudge(&why);
9853                let nudged = has_context(&specs[i], &seats[i], src.sessions);
9854                let prompt = if nudged {
9855                    nudge
9856                } else {
9857                    format!("{full}\n\n---\n\n{nudge}")
9858                };
9859                (
9860                    prompt,
9861                    retry_budget(src.timeout, nudged),
9862                    format!("{}-retry{}", src.stem, nudges[i]),
9863                )
9864            };
9865            batch.push(SeatJob {
9866                spec: specs[i].clone(),
9867                seat: seats[i].clone(),
9868                cwd: src.cwd.clone(),
9869                prompt,
9870                timeout,
9871                allow_write: src.allow_write,
9872                sessions: src.sessions,
9873                artifacts: src.artifacts.clone(),
9874                stem,
9875                handover: None,
9876            });
9877        }
9878
9879        if !renudged.is_empty() {
9880            state.event(
9881                ctx.node,
9882                format!("retry {round}: re-asking {}", renudged.join(", ")),
9883            );
9884        }
9885        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9886        let mut still = Vec::new();
9887        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9888            seats[i] = seat;
9889            // No agent was asked: nothing to hand over, nudge, remember or
9890            // charge. The caller sees `resource_block` and stops the node.
9891            if let AgentOutcome::ResourceBlocked(why) = &out {
9892                done[i] = Some(Err(anyhow::anyhow!("{why}")));
9893                continue;
9894            }
9895            let class = FailClass::of(&out);
9896            // A dropped stream is nudged first (the conversation is still
9897            // there to pick up); only a seat whose nudges are spent hands over.
9898            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9899                && nudges[i] < retries
9900                && !roster.is_empty();
9901            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9902                let next = should_hand_over(prev[i].as_ref(), &cur)
9903                    .then(|| next_agent(i, &tried[i], &specs))
9904                    .flatten();
9905                if carry {
9906                    let h = state
9907                        .seat_history
9908                        .entry(originals[i].seat.key.clone())
9909                        .or_default();
9910                    h.failed.insert(specs[i].id.clone());
9911                    h.last_fail = Some(cur.clone());
9912                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9913                        h.last_ok = None;
9914                    }
9915                    // Saved before the next agent is asked, so a restart in
9916                    // between does not forget who failed.
9917                    if let Err(e) = state.save() {
9918                        tracing::warn!("could not persist a seat's failure history: {e:#}");
9919                    }
9920                }
9921                if let Some(next) = next {
9922                    record_handover(
9923                        state,
9924                        ctx.node,
9925                        &originals[i].seat.key,
9926                        &specs[i].id,
9927                        &next.id,
9928                        &cur,
9929                        &fail_reason(&out),
9930                    );
9931                    tried[i].insert(next.id.clone());
9932                    prev[i] = Some(cur.clone());
9933                    seats[i] =
9934                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9935                    specs[i] = next;
9936                    fresh[i] = Some(cur.stem_word().to_owned());
9937                    nudges[i] = 0;
9938                    done[i] = Some(Err(anyhow::anyhow!(
9939                        "handed over after: {}",
9940                        fail_reason(&out)
9941                    )));
9942                    still.push(i);
9943                    continue;
9944                }
9945            }
9946            if carry
9947                && !nudge_first
9948                && let Some(cur) = class.clone()
9949            {
9950                // The chain ended here (no successor, or a repeated failure).
9951                let h = state
9952                    .seat_history
9953                    .entry(originals[i].seat.key.clone())
9954                    .or_default();
9955                h.failed.insert(specs[i].id.clone());
9956                h.last_fail = Some(cur);
9957            }
9958            let parsed = match out {
9959                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9960                AgentOutcome::Quota(o) => {
9961                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9962                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9963                }
9964                // Not a parseable answer: the nudge loop re-asks it, which is
9965                // exactly what a dropped stream needs. Just don't hand its raw
9966                // error JSON to `extract_json`.
9967                AgentOutcome::Dropped(o) => {
9968                    let why = o
9969                        .dropped
9970                        .as_ref()
9971                        .map(|d| d.why.as_str())
9972                        .unwrap_or("the CLI ended the stream without delivering its answer");
9973                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9974                }
9975                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9976                    Err(anyhow::anyhow!(e))
9977                }
9978            };
9979            let quota = class == Some(FailClass::Quota);
9980            let failed = parsed.is_err();
9981            done[i] = Some(parsed);
9982            if carry && !failed {
9983                let h = state
9984                    .seat_history
9985                    .entry(originals[i].seat.key.clone())
9986                    .or_default();
9987                h.failed.remove(&specs[i].id);
9988                h.last_ok = Some(specs[i].id.clone());
9989                h.last_fail = None;
9990            }
9991            // Do not re-ask a rate-limited seat (quota) — a retry is known to
9992            // fail the same way; and never re-ask a seat that already parsed.
9993            // A failed agent that still has a successor is not re-asked
9994            // either: the handover was its remedy and has just been refused
9995            // (the chain stops on a repeated failure class). A seat with no
9996            // successor left (a single-agent roster, the roster's tail, or an
9997            // empty roster) keeps the same-agent nudge, bounded by `retries`.
9998            let agent_failure = class.is_some()
9999                && !nudge_first
10000                && !roster.is_empty()
10001                && next_agent(i, &tried[i], &specs).is_some();
10002            if failed && !quota && !agent_failure && nudges[i] < retries {
10003                nudges[i] += 1;
10004                still.push(i);
10005            }
10006        }
10007        pending = still;
10008    }
10009
10010    // One loss per seat whose chain ended on a quota: the intermediate ones
10011    // were absorbed by a handover and are not losses.
10012    for (i, q) in last_quota.into_iter().enumerate() {
10013        if let Some(reset) = q {
10014            losses.push(QuotaLoss {
10015                seat: originals[i].seat.key.clone(),
10016                node: ctx.node.to_owned(),
10017                at: Timestamp::now(),
10018                reset,
10019            });
10020        }
10021    }
10022
10023    seats
10024        .into_iter()
10025        .zip(done)
10026        .zip(nudges)
10027        .map(|((seat, res), attempts)| {
10028            (
10029                seat,
10030                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10031                attempts,
10032            )
10033        })
10034        .collect()
10035}
10036
10037/// Did the wave end without asking any agent (see [`block_wave`])?
10038fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10039    results
10040        .iter()
10041        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10042}
10043
10044/// Settle a wave that never spawned anything: every job answers
10045/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10046/// else about the seat (session, history, quota) is touched.
10047fn block_wave(
10048    jobs: Vec<SeatJob>,
10049    state: &mut RunState,
10050    node: &str,
10051    attempt: usize,
10052    why: &str,
10053) -> Vec<(usize, SeatState, AgentOutcome)> {
10054    state.resource_wait = None;
10055    state
10056        .active
10057        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10058    if let Err(e) = state.save() {
10059        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10060    }
10061    jobs.into_iter()
10062        .enumerate()
10063        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10064        .collect()
10065}
10066
10067/// Acquire the shared build cache's lease, waiting out contention within
10068/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10069/// unbounded wait is never acceptable).
10070///
10071/// A first, non-blocking check happens before ever waiting; if it finds the
10072/// lease busy, that fact is logged as a `verify` event *and* flushed with
10073/// [`RunState::save`] immediately — not only once the wait finally succeeds
10074/// or gives up — so a `magi show` run by a different process while this one
10075/// is still waiting reads a `run.json` that says so, rather than whatever it
10076/// looked like before the wait started. The same applies to the terminal
10077/// failure: logged and saved before this returns `Err`, so a caller that
10078/// could not get the lease at all still leaves a legible record of why.
10079async fn acquire_cache_lease(
10080    state: &mut RunState,
10081    cache_dir: &Path,
10082    owner: &crate::cache::Owner,
10083    budget: Duration,
10084    context: &str,
10085    cancel: &(dyn Fn() -> bool + Sync),
10086) -> Result<Option<crate::cache::Guard>> {
10087    let home = crate::run::home();
10088    let started = Instant::now();
10089    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10090        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10091        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10092        Err(e) => {
10093            state.event(
10094                "verify",
10095                format!("{context}: could not check the shared build cache: {e:#}"),
10096            );
10097            if let Err(e2) = state.save() {
10098                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10099            }
10100            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10101        }
10102    };
10103    state.event(
10104        "verify",
10105        format!(
10106            "{context}: waiting for the shared build cache at {} ({})",
10107            cache_dir.display(),
10108            busy.describe()
10109        ),
10110    );
10111    // Seats of this node are waiting, not running: say so, so the countdown a
10112    // reader sees only ever measures an agent that has been spawned.
10113    state.resource_wait = Some(crate::run::ResourceWait {
10114        node: context.to_owned(),
10115        kind: "shared build cache".to_owned(),
10116        holder: busy.describe(),
10117        since: Timestamp::now(),
10118        limit_secs: budget.as_secs(),
10119    });
10120    if let Err(e) = state.save() {
10121        tracing::warn!("could not persist a cache wait: {e:#}");
10122    }
10123    let remaining = budget.saturating_sub(started.elapsed());
10124    let waited = crate::cache::wait_for_unless(
10125        &home,
10126        cache_dir,
10127        owner,
10128        remaining,
10129        Duration::from_secs(5),
10130        cancel,
10131    )
10132    .await;
10133    state.resource_wait = None;
10134    match waited {
10135        Ok(g) => Ok(g),
10136        Err(e) => {
10137            state.event("verify", format!("{context}: {e:#}"));
10138            if let Err(e2) = state.save() {
10139                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10140            }
10141            Err(e)
10142        }
10143    }
10144}
10145
10146/// The lease state could not be read at all (unreadable lease file, a
10147/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10148/// known to be contended, so a wave degrades to running without a shared cache
10149/// instead of stopping on a hold that [`crate::triage`] could never release.
10150#[derive(Debug)]
10151struct CacheCheckError(String);
10152
10153impl std::fmt::Display for CacheCheckError {
10154    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10155        f.write_str(&self.0)
10156    }
10157}
10158
10159impl std::error::Error for CacheCheckError {}
10160
10161/// Run `body` — a verify command batch — while holding the shared build
10162/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10163/// never interleave with another borrower's build against the same
10164/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10165/// timeout, or a human's own `magi review`. See the `cache` module doc for
10166/// why this matters more than Cargo's own per-target locking covers — two
10167/// *different* worktrees building the same package name/version into one
10168/// cache directory is a staleness bug, not a lock contention one.
10169///
10170/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10171/// agent waves do. Repository scheduling also has its own bounded,
10172/// cancellable resource-wait policy. Contention must not consume the
10173/// verification budget: `body` receives the full `budget` after both leases
10174/// are acquired and the cache is fresh. Without a shared cache, only
10175/// repository coordination is acquired before handing over the full budget.
10176///
10177/// A lease that cannot be acquired within the resource wait is reported as a single
10178/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10179/// verification — the same shape a spawn failure already takes in
10180/// [`run_commands`], so a caller need not special-case it.
10181#[allow(clippy::too_many_arguments)]
10182async fn with_cache_lease<'s, F, Fut>(
10183    state: &'s mut RunState,
10184    cache_dir: Option<&Path>,
10185    node: &str,
10186    seat: &str,
10187    worktree: &Path,
10188    head: &str,
10189    budget: Duration,
10190    context: &str,
10191    wait: &crate::repo_lock::WaitPolicy,
10192    body: F,
10193) -> Result<(Vec<CommandOutcome>, bool)>
10194where
10195    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10196    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10197{
10198    // Match editing waves: cache first, repository second. In particular,
10199    // pre_gate must never hold the repository while waiting for a wave's cache.
10200    let started = Instant::now();
10201    let home = crate::run::home();
10202    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10203    let guard = if let Some(cache_dir) = cache_dir {
10204        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10205        match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10206            wait.cancelled()
10207        })
10208        .await
10209        .and_then(|g| {
10210            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10211        }) {
10212            Ok(g) => Some(g),
10213            Err(e) => {
10214                return Ok((
10215                    vec![CommandOutcome {
10216                        command: "(waiting for the shared build cache)".into(),
10217                        code: None,
10218                        output_tail: e.to_string(),
10219                        duration_ms: started.elapsed().as_millis() as u64,
10220                        resource_blocked: true,
10221                    }],
10222                    false,
10223                ));
10224            }
10225        }
10226    } else {
10227        None
10228    };
10229    let _coordination = coordinate_operation(state, worktree, wait).await?;
10230    if let Some(cache_dir) = cache_dir {
10231        let identity = crate::cache::Identity::new(worktree, head);
10232        if let Err(e) = crate::cache::ensure_fresh(
10233            &home,
10234            cache_dir,
10235            &identity,
10236            &state.config.verify.cargo_manifests(),
10237        ) {
10238            state.event(
10239                "verify",
10240                format!(
10241                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10242                    worktree.display(),
10243                    short(head)
10244                ),
10245            );
10246            if let Some(guard) = guard {
10247                guard.release();
10248            }
10249            return Ok((
10250                vec![CommandOutcome {
10251                    command: "(confirming the shared build cache is fresh)".into(),
10252                    code: None,
10253                    output_tail: e.to_string(),
10254                    duration_ms: started.elapsed().as_millis() as u64,
10255                    resource_blocked: true,
10256                }],
10257                false,
10258            ));
10259        }
10260    }
10261    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10262    if !timed_out_pids.is_empty() {
10263        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10264    }
10265    // Drop repository coordination before releasing the cache, reversing the
10266    // acquisition order. A cancelled/error future also drops both guards.
10267    drop(_coordination);
10268    if let Some(guard) = guard {
10269        guard.release();
10270    }
10271    Ok((outcomes, retried))
10272}
10273
10274/// Poll `pids` — commands [`run_commands`] reports as still running when its
10275/// own timeout elapsed — until every one is confirmed gone, or
10276/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10277///
10278/// Real confirmation where confirmation is possible, not a substitute for
10279/// full process-tree observation: a grandchild the timed-out process spawned
10280/// and that survives independently of it is invisible to a pid check the
10281/// same way it always was, and continuing to observe and collect *that*
10282/// stays a different piece of work with its own owner. This only narrows a
10283/// fixed blind wait into an actual check of the pids this process does know
10284/// about.
10285async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10286    wait_for_pids_with(
10287        pids,
10288        crate::proc::pid_alive,
10289        LEASE_RELEASE_POLL,
10290        LEASE_RELEASE_MAX_WAIT,
10291    )
10292    .await;
10293}
10294
10295/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10296/// interval and ceiling supplied by the caller, so the polling *logic* -
10297/// returns as soon as every pid reports dead, gives up at the ceiling
10298/// otherwise - is testable on millisecond durations without asking the real
10299/// OS about a pid at all.
10300async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10301    pids: &[u32],
10302    alive: F,
10303    poll: Duration,
10304    max_wait: Duration,
10305) {
10306    let deadline = Instant::now() + max_wait;
10307    loop {
10308        if pids.iter().all(|&pid| !alive(pid)) {
10309            return;
10310        }
10311        if Instant::now() >= deadline {
10312            return;
10313        }
10314        tokio::time::sleep(poll).await;
10315    }
10316}
10317
10318/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10319/// admission that it could not even get a verify command to run, as opposed
10320/// to evidence the command actually produced? A caller that would otherwise
10321/// read a resource-blocked outcome as a red command must check this first:
10322/// see [`Runner::gate`], which retries rather than records `Blocked` when
10323/// this is true.
10324fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10325    outcomes.iter().any(|o| o.resource_blocked)
10326}
10327
10328/// What [`Runner::gate_fix_round`] decided.
10329pub(crate) enum GateFix {
10330    /// The tree changed and `verify.e2e` is still green: run the gate again.
10331    Retry,
10332    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10333    /// last failure stands and the run ends blocked.
10334    Stop,
10335    /// `verify.e2e` could not run after the fix (magi's own contention):
10336    /// decide nothing now, a later reentry retries.
10337    Defer,
10338}
10339
10340/// Is every red command in `outcomes` an ordinary failure the code could
10341/// explain: it ran, exited non-zero, and said something?
10342///
10343/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10344/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10345/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10346/// what the command is stays the gate's business.
10347fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10348    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10349    red.peek().is_some()
10350        && red.all(|o| {
10351            !o.resource_blocked
10352                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10353                && !o.output_tail.trim().is_empty()
10354        })
10355}
10356
10357/// Describe one verify command's outcome for the event log, distinguishing a
10358/// build/link failure — the toolchain never produced a binary to run — from
10359/// an actual test failure, since only the latter is a verdict on the patch.
10360fn e2e_outcome_label(o: &CommandOutcome) -> String {
10361    if o.ok() {
10362        return "pass".to_owned();
10363    }
10364    let reason = if o.build_failed() {
10365        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10366    } else {
10367        format!("FAIL ({:?})", o.code)
10368    };
10369    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10370}
10371
10372/// Run `verify.e2e`, retrying once if the first attempt could not build or
10373/// link — a build/link failure is frequently a race against a shared
10374/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10375/// `verify` event per command, tagged with `context` (normally `"round N"`)
10376/// so the two call sites that need this — the ordinary per-round leg in
10377/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10378/// it will ever call a round green — read identically in the event log.
10379async fn run_e2e_with_retry(
10380    state: &mut RunState,
10381    shell: &[String],
10382    commands: &[String],
10383    worktree: &Path,
10384    timeout: Duration,
10385    context: &str,
10386) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10387    let (mut e2e, mut timed_out_pids) = run_commands(
10388        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10389    )
10390    .await;
10391    for o in &e2e {
10392        state.event(
10393            "verify",
10394            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10395        );
10396    }
10397    // A build/link failure is not a verdict on the patch — it is frequently a
10398    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10399    // one retry before letting a red like that decide the round.
10400    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10401    if verify_retried {
10402        state.event(
10403            "verify",
10404            format!(
10405                "{context}: verify could not build/link, not a test result — retrying once \
10406                 before concluding"
10407            ),
10408        );
10409        let retried = run_commands(
10410            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10411        )
10412        .await;
10413        e2e = retried.0;
10414        // Both attempts' timeouts matter, not just the last one: the first
10415        // attempt's descendants may still be alive alongside the retry's.
10416        timed_out_pids.extend(retried.1);
10417        for o in &e2e {
10418            state.event(
10419                "verify",
10420                format!(
10421                    "{context}: retry `{}` -> {}",
10422                    o.command,
10423                    e2e_outcome_label(o)
10424                ),
10425            );
10426        }
10427    }
10428    (e2e, verify_retried, timed_out_pids)
10429}
10430
10431/// Run configured shell commands in `cwd`, in order. The second element is
10432/// the pid of every command that hit `timeout` and was still running when
10433/// this stopped waiting on it (best-effort: `None` when the platform did not
10434/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10435/// that releases a shared resource afterward needs to know.
10436///
10437/// Records `task` into [`RunState::active`] at every command boundary
10438/// (`RunState::task_command`) and clears it once the whole list has run
10439/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10440/// for minutes with no seat and no output of its own to show for it (see
10441/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10442/// apart from "not yet run" without this), and this is the only place that
10443/// knows which command is running right now and how many are left. Three
10444/// saves per command — start, not per second — matching the same "only at a
10445/// boundary" rule [`wave`] already follows for seats.
10446#[allow(clippy::too_many_arguments)]
10447async fn run_commands(
10448    state: &mut RunState,
10449    node: &str,
10450    task: &str,
10451    attempt: usize,
10452    shell: &[String],
10453    commands: &[String],
10454    cwd: &Path,
10455    timeout: Duration,
10456) -> (Vec<CommandOutcome>, Vec<u32>) {
10457    if commands.is_empty() {
10458        // Nothing to mark as running and nothing to clear — an empty list
10459        // means "not configured", and touching `active` (or the disk) over
10460        // that would be a write for every round of a repo with no
10461        // `verify.e2e` / `verify.gate` commands at all.
10462        return (Vec::new(), Vec::new());
10463    }
10464    let mut out = Vec::new();
10465    let mut timed_out_pids = Vec::new();
10466    let total = commands.len();
10467    for (idx, command) in commands.iter().enumerate() {
10468        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10469        if let Err(e) = state.save() {
10470            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10471        }
10472        let started = Instant::now();
10473        let mut cmd = tokio::process::Command::new(&shell[0]);
10474        cmd.quiet();
10475        cmd.args(&shell[1..])
10476            .arg(command)
10477            .current_dir(cwd)
10478            .stdin(std::process::Stdio::null())
10479            .stdout(std::process::Stdio::piped())
10480            .stderr(std::process::Stdio::piped())
10481            .kill_on_drop(true);
10482        let mut outputs = Vec::new();
10483        let protected = (|| -> Result<()> {
10484            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10485                outputs.push(guard);
10486            }
10487            if let Some(cache) = state.config.cache_dir() {
10488                outputs.push(crate::retention::OutputLease::cache(
10489                    &crate::run::home(),
10490                    &cache,
10491                    false,
10492                )?);
10493            }
10494            Ok(())
10495        })();
10496        if let Err(e) = protected {
10497            out.push(CommandOutcome {
10498                command: command.clone(),
10499                code: None,
10500                output_tail: format!("output ownership unavailable: {e:#}"),
10501                duration_ms: 0,
10502                resource_blocked: true,
10503            });
10504            continue;
10505        }
10506        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10507        let (code, body) = match spawned {
10508            Ok(mut running) => {
10509                let pid = running.child().id();
10510                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10511                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10512                let mut paths = vec![cwd.to_owned()];
10513                if let Some(cache) = state.config.cache_dir() {
10514                    paths.push(cache);
10515                }
10516                let mut pressure = None;
10517                let status = tokio::select! {
10518                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10519                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10520                        pressure = Some(reason); None
10521                    }
10522                };
10523                let (code, failure) = match status {
10524                    Some(Ok(Ok(status))) => (
10525                        status.code(),
10526                        status
10527                            .code()
10528                            .is_none()
10529                            .then(|| format!("command terminated: {status}")),
10530                    ),
10531                    Some(Ok(Err(error))) => {
10532                        (None, Some(format!("failed waiting for command: {error}")))
10533                    }
10534                    Some(Err(_)) => {
10535                        // Leased cleanup confirms process-tree death itself.
10536                        // Unleased commands still need the caller's PID check
10537                        // after kill-on-drop, before releasing cache ownership.
10538                        if !running.coordinated()
10539                            && let Some(pid) = pid
10540                        {
10541                            timed_out_pids.push(pid);
10542                        }
10543                        (
10544                            None,
10545                            Some(format!("timed out after {}s", timeout.as_secs())),
10546                        )
10547                    }
10548                    None => (None, None),
10549                };
10550                running.finish().await;
10551                if let Some(reason) = pressure {
10552                    state.resource_block = Some(crate::run::ResourceBlock {
10553                        node: node.to_owned(),
10554                        kind: "disk pressure".to_owned(),
10555                        detail: reason.clone(),
10556                        at: Timestamp::now(),
10557                    });
10558                    out.push(CommandOutcome {
10559                        command: command.clone(),
10560                        code: None,
10561                        output_tail: reason,
10562                        duration_ms: started.elapsed().as_millis() as u64,
10563                        resource_blocked: true,
10564                    });
10565                    break;
10566                }
10567                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10568                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10569                if let Some(failure) = failure {
10570                    body.push('\n');
10571                    body.push_str(&failure);
10572                }
10573                (code, body)
10574            }
10575            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10576        };
10577        out.push(CommandOutcome {
10578            command: command.clone(),
10579            code,
10580            output_tail: tail(&body, OUTPUT_TAIL),
10581            duration_ms: started.elapsed().as_millis() as u64,
10582            resource_blocked: false,
10583        });
10584    }
10585    state.task_finished(task);
10586    if let Err(e) = state.save() {
10587        tracing::warn!("could not persist the end of {task}: {e:#}");
10588    }
10589    (out, timed_out_pids)
10590}
10591
10592/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10593/// section (`report::run`) and in the `merge` event this node records — for
10594/// the operator to run by hand.
10595///
10596/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10597/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10598/// merge commits", or "require linear history") rejects the push a `--no-ff`
10599/// merge would produce, which is exactly the guidance this function replaces.
10600/// `message`'s first line becomes the squash commit's subject, matching the
10601/// note `report::run` prints alongside this command — see that function for
10602/// why an explicit subject is not optional there.
10603fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10604    let repo = repo.display();
10605    match style {
10606        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10607        MergeStyle::Squash => {
10608            // The subject sits inside double quotes, and a title an agent
10609            // wrote may carry the characters that break out of them.
10610            let subject = message
10611                .lines()
10612                .next()
10613                .unwrap_or(branch)
10614                .replace(['\\', '"', '$', '`'], "");
10615            format!(
10616                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10617            )
10618        }
10619        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10620    }
10621}
10622
10623/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10624/// under the hood, rejects a `title` over 256 characters and the whole
10625/// command fails — no PR at all, for a run whose body was otherwise fine
10626/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10627/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10628/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10629/// this bound. It is a margin, not a guarantee — a title packed
10630/// with multi-unit characters could still in principle land close to the
10631/// edge, but a real task title's occasional emoji or accented letter fits
10632/// comfortably inside it.
10633const PR_TITLE_MAX: usize = 240;
10634
10635/// A pull request title from the opening line of `text`, or `None` when that
10636/// line is not English (GitHub text is) or has no letters.
10637///
10638/// A line within `max` is kept as is. A longer one is cut at the end of its
10639/// first sentence when that falls inside `max`, else at a word boundary with a
10640/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10641/// make every merely-truncated title look non-English). The language check
10642/// runs on the kept text before any mark is added, so only what GitHub will
10643/// show is judged: an English opening followed by non-ASCII far past the cut
10644/// still passes.
10645fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10646    let line = queue::first_line(text)?;
10647    let chars: Vec<char> = line.chars().collect();
10648    let (kept, mark) = if chars.len() <= max {
10649        (line.to_owned(), "")
10650    } else if let Some(end) = sentence_end(&chars, max) {
10651        (chars[..end].iter().collect::<String>(), "")
10652    } else {
10653        let room = max.saturating_sub(3);
10654        // Cut at the last space inside the room; when the char just past the
10655        // room is a space the room already ends on a word.
10656        let cut = if chars[room].is_whitespace() {
10657            room
10658        } else {
10659            chars[..room]
10660                .iter()
10661                .rposition(|c| c.is_whitespace())
10662                .unwrap_or(room)
10663        };
10664        let head: String = chars[..cut].iter().collect();
10665        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10666        (head.to_owned(), "...")
10667    };
10668    if kept.is_empty() {
10669        return None;
10670    }
10671    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10672        return None;
10673    }
10674    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10675        return None;
10676    }
10677    Some(format!("{kept}{mark}"))
10678}
10679
10680/// The char length of the first sentence of `chars` when it ends within `max`
10681/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10682/// abbreviations so `e.g. foo` does not end a title early.
10683fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10684    const MIN: usize = 20;
10685    for i in MIN..max.min(chars.len()) {
10686        if !matches!(chars[i], '.' | '!' | '?') {
10687            continue;
10688        }
10689        let Some(&next) = chars.get(i + 1) else {
10690            continue;
10691        };
10692        if !next.is_whitespace() {
10693            continue;
10694        }
10695        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10696        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10697            continue;
10698        }
10699        let word: String = chars[..i]
10700            .iter()
10701            .rev()
10702            .take_while(|c| !c.is_whitespace())
10703            .collect::<Vec<_>>()
10704            .into_iter()
10705            .rev()
10706            .collect();
10707        let word = word.to_ascii_lowercase();
10708        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10709            continue;
10710        }
10711        let end = chars[..i]
10712            .iter()
10713            .rposition(|c| !c.is_whitespace())
10714            .map_or(i, |p| p + 1);
10715        return Some(end);
10716    }
10717    None
10718}
10719
10720/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10721/// change: a title and a body describing what was *implemented*, not the task
10722/// that asked for it. A task reads as a request; a reader of the merged
10723/// history wants the change.
10724struct PrMessage {
10725    title: String,
10726    body: String,
10727}
10728
10729impl PrMessage {
10730    /// Title, blank line, body. The first line is the squash/merge commit
10731    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
10732    /// has to stay one sensible line.
10733    fn commit_message(&self) -> String {
10734        format!("{}\n\n{}", self.title, self.body)
10735    }
10736}
10737
10738/// The text after a leading `TITLE:` (any case) on `line`.
10739fn title_marker(line: &str) -> Option<&str> {
10740    let line = line.trim();
10741    let head = line.get(..6)?;
10742    head.eq_ignore_ascii_case("title:")
10743        .then(|| line[6..].trim())
10744}
10745
10746/// The implementer's own one-line title: the `TITLE:` line the implement
10747/// prompt asks for at the top of its SUMMARY. Candidate commits are all
10748/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
10749/// source, and a title that says as much is refused here too.
10750fn summary_title(summary: &str) -> Option<String> {
10751    let first = summary.lines().find(|l| !l.trim().is_empty())?;
10752    let raw = title_marker(first)?;
10753    if raw.is_empty() {
10754        return None;
10755    }
10756    let title = queue::title_from(raw, PR_TITLE_MAX);
10757    let lower = title.to_ascii_lowercase();
10758    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10759        return None;
10760    }
10761    Some(title)
10762}
10763
10764/// How `open_review`'s instruction begins; see [`landing_title`].
10765const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10766
10767/// Marker `open_review` gives a candidate that nothing in the roster wrote.
10768pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10769
10770/// Does this run review work that already existed, rather than implement a
10771/// task? Runs recorded before `reviewed_commits` existed carry only the
10772/// candidate marker.
10773fn is_review_run(state: &RunState) -> bool {
10774    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10775}
10776
10777/// The title of a review-only run: the subject of the oldest commit under
10778/// review. Later commits are usually fixups, and `instruction` is the review
10779/// prompt, which says nothing about the change. GitHub text is English, so a
10780/// non-ASCII or blank subject yields `None` and the caller's neutral title.
10781fn review_title(state: &RunState) -> Option<String> {
10782    english_subject(
10783        state.reviewed_commits.as_ref()?.first()?,
10784        state.config.graph.github_english(),
10785    )
10786}
10787
10788/// `raw` as a pull request title, or `None` when it is blank, not English
10789/// (GitHub text is), or one of magi's own candidate commit subjects.
10790fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10791    let raw = raw.trim();
10792    if raw.is_empty() {
10793        return None;
10794    }
10795    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10796        return None;
10797    }
10798    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10799        return None;
10800    }
10801    let title = queue::title_from(raw, PR_TITLE_MAX);
10802    let lower = title.to_ascii_lowercase();
10803    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10804        return None;
10805    }
10806    Some(title)
10807}
10808
10809/// What a review-only run's branch says about itself, read at the moment the
10810/// pull request is opened.
10811#[derive(Debug, Clone, PartialEq, Eq)]
10812struct BranchFacts {
10813    /// `(subject, body)` of each commit, oldest first.
10814    commits: Vec<(String, String)>,
10815    /// Trimmed `git diff --stat`.
10816    stat: String,
10817}
10818
10819/// Longest diff stat shown: this many file lines plus the summary line.
10820const STAT_FILE_LINES: usize = 25;
10821/// Cap on the commit list, well inside GitHub's 65536-character body limit.
10822const COMMITS_MAX_CHARS: usize = 20_000;
10823
10824/// Read the commits and diff stat of `base..branch`. `None` when git cannot
10825/// say or finds nothing, so the caller falls back to what the run recorded.
10826async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10827    let commits = git::commit_log(repo, base, branch).await.ok()?;
10828    if commits.is_empty() {
10829        return None;
10830    }
10831    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10832    let lines: Vec<&str> = stat.lines().collect();
10833    let stat = if lines.len() > STAT_FILE_LINES + 1 {
10834        let omitted = lines.len() - 1 - STAT_FILE_LINES;
10835        let more = format!(" ... {omitted} more file(s)");
10836        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10837        kept.push(&more);
10838        kept.push(lines[lines.len() - 1]);
10839        kept.join("\n")
10840    } else {
10841        lines.join("\n")
10842    };
10843    Some(BranchFacts { commits, stat })
10844}
10845
10846/// Defang what would break the surrounding markdown: a closing `</details>`
10847/// and a code fence.
10848fn markdown_safe(text: &str) -> String {
10849    text.replace("</details>", "&lt;/details&gt;")
10850        .replace("\x60\x60\x60", "~~~")
10851}
10852
10853fn neutral_title(state: &RunState, winner: char) -> String {
10854    format!(
10855        "chore: land candidate {} of run {}",
10856        winner.to_ascii_uppercase(),
10857        state.id
10858    )
10859}
10860
10861/// Reuse the graph's bounded fixer without starting another graph or driver.
10862/// Copy the working state back even on errors so spent rounds remain visible.
10863pub(crate) async fn repair_landing_gate(
10864    state: &mut RunState,
10865    winner: &Candidate,
10866    outcomes: &[CommandOutcome],
10867) -> Result<GateFix> {
10868    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10869        return Ok(GateFix::Stop);
10870    }
10871    let mut runner = Runner {
10872        roles: state.config.resolve_roles()?,
10873        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10874        state: state.clone(),
10875        pause: Pause::new(),
10876        interrupt: Pause::new(),
10877    };
10878    let result = runner.gate_fix_round(winner, outcomes).await;
10879    *state = runner.state;
10880    result
10881}
10882
10883/// Recheck a repair's e2e requirement across failures, resource deferrals and
10884/// process interruptions. A successful result only applies to its exact SHA.
10885pub(crate) async fn check_gate_repair_e2e(
10886    state: &mut RunState,
10887    winner: &Candidate,
10888    wait: &crate::repo_lock::WaitPolicy,
10889) -> Result<GateFix> {
10890    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10891        return Ok(GateFix::Retry);
10892    }
10893    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10894    if state
10895        .gate_repair_e2e
10896        .as_ref()
10897        .is_some_and(|r| r.head == head && r.passed)
10898    {
10899        return Ok(GateFix::Retry);
10900    }
10901    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10902        head: head.clone(),
10903        passed: false,
10904        outcomes: Vec::new(),
10905    });
10906    state.save()?;
10907    let commands = state.config.verify.e2e.clone();
10908    let outcomes = if commands.is_empty() {
10909        Vec::new()
10910    } else {
10911        let shell = state.config.shell();
10912        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10913        let cache_dir = state.config.cache_dir();
10914        let (outcomes, _) = with_cache_lease(
10915            state,
10916            cache_dir.as_deref(),
10917            "e2e",
10918            "e2e",
10919            &winner.worktree,
10920            &head,
10921            timeout,
10922            "gate repair e2e",
10923            wait,
10924            |state, budget| async move {
10925                run_e2e_with_retry(
10926                    state,
10927                    &shell,
10928                    &commands,
10929                    &winner.worktree,
10930                    budget,
10931                    "gate repair e2e",
10932                )
10933                .await
10934            },
10935        )
10936        .await?;
10937        outcomes
10938    };
10939    let deferred = verify_inconclusive(&outcomes);
10940    let passed = outcomes.iter().all(CommandOutcome::ok)
10941        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10942    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10943        head,
10944        passed,
10945        outcomes,
10946    });
10947    state.save()?;
10948    if deferred {
10949        Ok(GateFix::Defer)
10950    } else if !passed {
10951        state.event(
10952            "gate",
10953            "verify.e2e failed after gate repair; nothing published",
10954        );
10955        Ok(GateFix::Stop)
10956    } else {
10957        Ok(GateFix::Retry)
10958    }
10959}
10960
10961/// Run the same configured gate for a landing head and a fresh graph.
10962pub(crate) async fn run_landing_gate(
10963    state: &mut RunState,
10964    winner: &Candidate,
10965    wait: &crate::repo_lock::WaitPolicy,
10966) -> Result<Vec<CommandOutcome>> {
10967    let shell = state.config.shell();
10968    let gate_commands = state.config.verify.gate.clone();
10969    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10970    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10971        Vec::new()
10972    } else {
10973        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10974        let cache_dir = state.config.cache_dir();
10975        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10976        let (outcomes, _) = with_cache_lease(
10977            state,
10978            cache_dir.as_deref(),
10979            "gate",
10980            "gate",
10981            &winner.worktree,
10982            &head,
10983            timeout,
10984            "final gate",
10985            wait,
10986            |state, budget| {
10987                let shell = shell.clone();
10988                let gate_commands = gate_commands.clone();
10989                let winner = winner.clone();
10990                async move {
10991                    Runner::run_pre_gate(state, &winner).await;
10992                    let (outcomes, timed_out_pids) = run_commands(
10993                        state,
10994                        "gate",
10995                        "gate",
10996                        0,
10997                        &shell,
10998                        &gate_commands,
10999                        &winner.worktree,
11000                        budget,
11001                    )
11002                    .await;
11003                    (outcomes, false, timed_out_pids)
11004                }
11005            },
11006        )
11007        .await?;
11008        outcomes
11009    };
11010    if outcomes.is_empty() {
11011        // Nothing configured to check — distinct from every other
11012        // silence in this run's event log, since an empty `gate` alone
11013        // no longer says whether the gate ran at all (see
11014        // `RunState::gate_ran`'s own doc).
11015        state.event("gate", "no gate commands configured; not tested");
11016    }
11017    for o in &outcomes {
11018        state.event(
11019            "gate",
11020            format!(
11021                "`{}` -> {}",
11022                o.command,
11023                if o.ok() {
11024                    "pass".to_owned()
11025                } else {
11026                    format!(
11027                        "FAIL ({:?})\n{}",
11028                        o.code,
11029                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11030                    )
11031                }
11032            ),
11033        );
11034    }
11035    Ok(outcomes)
11036}
11037
11038/// The pull request title to hand to `land::merge_subject`. A review-only run
11039/// opened by an earlier build titled its pull request with the review prompt;
11040/// that title is dropped (empty, so the fallback applies) rather than landed.
11041/// Any other title, including an operator's rename, passes through untouched,
11042/// and so does every title of a run that implements a task.
11043pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11044    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11045        ""
11046    } else {
11047        pr_title
11048    }
11049}
11050
11051/// What the squash subject falls back to when the pull request title is empty
11052/// or candidate-shaped: for a review-only run the derived title, never the
11053/// review prompt held in `instruction`.
11054pub fn landing_subject_source(state: &RunState) -> String {
11055    if is_review_run(state) {
11056        let winner = state.candidates.first().map_or('A', |c| c.label);
11057        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11058    }
11059    state.instruction.clone()
11060}
11061
11062/// `summary` without its `TITLE:` line, which the pull request title already
11063/// carries.
11064fn summary_without_title(summary: &str) -> String {
11065    let mut lines = summary.trim().lines().peekable();
11066    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11067        lines.next();
11068    }
11069    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11070}
11071
11072/// The pull request title and body for the winning candidate.
11073///
11074/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11075/// the task's own opening line via [`queue::title_from`] when there is none.
11076/// `state.instruction` can open with blank lines (`task_text` only rejects a
11077/// body that is blank *entirely*), which `title_from` skips.
11078///
11079/// Publishing uses an independent description generated from the final diff
11080/// and structured evidence. Internal summaries and reports remain local.
11081/// Non-publishing previews may display the original task and candidate summary.
11082/// The footer retains the run and candidate provenance.
11083#[cfg(test)]
11084fn pr_message(state: &RunState, winner: char) -> PrMessage {
11085    pr_message_with(state, winner, None)
11086}
11087
11088/// [`pr_message`] with what the branch of a review-only run says about itself.
11089/// `facts` is ignored for a run that implements a task.
11090#[cfg(test)]
11091fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11092    let raw = pr_message_raw(
11093        state,
11094        winner,
11095        facts,
11096        crate::pr_outcome::FinalHead::Unreported,
11097        ChangeNote::NotAsked,
11098    );
11099    let id = crate::scrub::Identity::current();
11100    PrMessage {
11101        title: crate::scrub::scrub(&raw.title, &id),
11102        body: crate::scrub::scrub(&raw.body, &id),
11103    }
11104}
11105
11106/// What the public description may say about the change itself.
11107#[derive(Debug, Clone, Copy)]
11108enum ChangeNote<'a> {
11109    /// A local preview or non-publishing merge mode: no public generation needed.
11110    NotAsked,
11111    /// A summary written from the final diff.
11112    Summary(&'a crate::run::ChangeSummary),
11113    /// A summary was asked for and could not be produced; the text says why.
11114    Unavailable(&'a str),
11115}
11116
11117fn pr_message_raw(
11118    state: &RunState,
11119    winner: char,
11120    facts: Option<&BranchFacts>,
11121    head: crate::pr_outcome::FinalHead<'_>,
11122    change: ChangeNote<'_>,
11123) -> PrMessage {
11124    if !matches!(change, ChangeNote::NotAsked) {
11125        let (title, mut body) = match change {
11126            ChangeNote::Summary(c) => (
11127                c.title.clone(),
11128                format!("## Summary\n\n{}\n\n", c.body.trim()),
11129            ),
11130            ChangeNote::Unavailable(why) => (
11131                neutral_title(state, winner),
11132                format!(
11133                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11134                ),
11135            ),
11136            ChangeNote::NotAsked => unreachable!(),
11137        };
11138        body.push_str(&public_pr_evidence(state, head));
11139        body.push_str(&format!(
11140            "\n---\nmagi:run/{} magi:candidate-{}\n",
11141            state.id,
11142            winner.to_ascii_lowercase()
11143        ));
11144        return PrMessage { title, body };
11145    }
11146    pr_preview(state, winner, facts, head)
11147}
11148
11149/// Append public facts from the run records after every prose generation,
11150/// including posting-gate rewrites. The agent cannot drop this provenance.
11151fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11152    let mut body = crate::pr_outcome::review_rounds(state);
11153    body.push_str(&crate::pr_outcome::final_verification(state, head));
11154    let open = state.last_round_findings();
11155    if !open.is_empty() {
11156        body.push_str("## Open review findings\n\n");
11157        for f in open {
11158            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11159        }
11160        body.push('\n');
11161    }
11162    if let Some(c) = state.change_summary.as_ref()
11163        && head == crate::pr_outcome::FinalHead::At(&c.head)
11164    {
11165        body.push_str(&format!(
11166            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11167            c.base, c.head
11168        ));
11169    }
11170    body
11171}
11172
11173/// Local-only preview of the operator's task and candidate summary.
11174fn pr_preview(
11175    state: &RunState,
11176    winner: char,
11177    facts: Option<&BranchFacts>,
11178    head: crate::pr_outcome::FinalHead<'_>,
11179) -> PrMessage {
11180    let summary = state
11181        .candidates
11182        .iter()
11183        .find(|c| c.label == winner)
11184        .map(|c| c.summary.as_str())
11185        .unwrap_or_default();
11186    // The fallback is the operator's own words and may not be English; GitHub
11187    // text always is, so a non-English task gets a neutral title instead.
11188    let review = is_review_run(state);
11189    let title = if review {
11190        facts
11191            .and_then(|f| {
11192                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11193            })
11194            .or_else(|| review_title(state))
11195            .or_else(|| {
11196                state
11197                    .candidates
11198                    .iter()
11199                    .find(|c| c.label == winner)
11200                    .filter(|c| !c.branch.starts_with("magi/"))
11201                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11202            })
11203            .unwrap_or_else(|| neutral_title(state, winner))
11204    } else {
11205        summary_title(summary).unwrap_or_else(|| {
11206            english_title(
11207                &state.instruction,
11208                PR_TITLE_MAX,
11209                state.config.graph.github_english(),
11210            )
11211            .unwrap_or_else(|| neutral_title(state, winner))
11212        })
11213    };
11214
11215    let mut body = String::new();
11216    let what = summary_without_title(summary);
11217    if !what.is_empty() {
11218        body.push_str("## Summary\n\n");
11219        body.push_str(&what);
11220        body.push_str("\n\n");
11221    }
11222
11223    // Review history uses structured facts across all rounds, never reports.
11224    body.push_str(&crate::pr_outcome::review_rounds(state));
11225    body.push_str(&crate::pr_outcome::final_verification(state, head));
11226    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11227
11228    let open = state.open_findings();
11229    if !open.is_empty() {
11230        body.push_str("## Open review findings\n\n");
11231        for f in &open {
11232            body.push_str(&format!(
11233                "- `{}` [{:?}] {}\n",
11234                f.id, f.severity, "Context retained in local review records."
11235            ));
11236        }
11237        body.push('\n');
11238    }
11239
11240    if let Some(fix) = fix
11241        && !fix.rejected.is_empty()
11242    {
11243        body.push_str("## Declined by the fixer\n\n");
11244        for r in &fix.rejected {
11245            body.push_str(&format!(
11246                "- `{}`: {}\n",
11247                r.id, "Reason retained in local review records."
11248            ));
11249        }
11250        body.push('\n');
11251    }
11252
11253    if review {
11254        // The review prompt is not the task; list what the branch carries.
11255        body.push_str("## Commits under review\n\n");
11256        if let Some(facts) = facts {
11257            let mut left = COMMITS_MAX_CHARS;
11258            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11259                let mut entry = format!("- {}\n", markdown_safe(subject));
11260                for l in markdown_safe(text).lines() {
11261                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11262                }
11263                if left == 0 {
11264                    body.push_str(&format!(
11265                        "- ... {} more commit(s)\n",
11266                        facts.commits.len() - i
11267                    ));
11268                    break;
11269                }
11270                if entry.len() > left {
11271                    // Even the first commit is cut: one huge body must not
11272                    // push the whole description past GitHub's limit.
11273                    let mut end = left;
11274                    while !entry.is_char_boundary(end) {
11275                        end -= 1;
11276                    }
11277                    entry.truncate(end);
11278                    entry.push_str("\n  ... (truncated)\n");
11279                    left = 0;
11280                } else {
11281                    left -= entry.len();
11282                }
11283                body.push_str(&entry);
11284            }
11285            if !facts.stat.trim().is_empty() {
11286                body.push_str(&format!(
11287                    "\n## Diff stat\n\n```\n{}\n```\n",
11288                    markdown_safe(facts.stat.trim())
11289                ));
11290            }
11291        } else {
11292            match &state.reviewed_commits {
11293                Some(subjects) => {
11294                    for s in subjects {
11295                        body.push_str(&format!("- {}\n", s.trim()));
11296                    }
11297                }
11298                None => {
11299                    // An older run kept only the prompt, with the commit list
11300                    // after its first paragraph.
11301                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11302                    body.push_str(rest.trim());
11303                    body.push('\n');
11304                }
11305            }
11306        }
11307    } else {
11308        let task = state.instruction.trim();
11309        let task = if task.is_empty() {
11310            "(empty task)"
11311        } else {
11312            task
11313        };
11314        body.push_str(&format!(
11315            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11316            task.replace("</details>", "&lt;/details&gt;")
11317        ));
11318    }
11319
11320    body.push_str(&format!(
11321        "\n---\nmagi:run/{} magi:candidate-{}\n",
11322        state.id,
11323        winner.to_ascii_lowercase()
11324    ));
11325
11326    PrMessage { title, body }
11327}
11328
11329/// The task with what the repository says about the existing work it names
11330/// appended, so an implementer knows what it started from and what it must
11331/// not redo. Unchanged when the task names nothing.
11332fn seeded_instruction(state: &RunState) -> String {
11333    match refs::describe(&state.seeds) {
11334        Some(facts) => format!(
11335            "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11336             Candidates start from the unmerged branch named above, when there \
11337             is one, and carry any unmerged commit named by sha as a \
11338             cherry-pick. Check that this is what the task meant before \
11339             building on it.",
11340            state.instruction
11341        ),
11342        None => state.instruction.clone(),
11343    }
11344}
11345
11346/// Does the winner have no commits ahead of the base it would land on?
11347/// Any failure to find out reads as "not empty": the merge then behaves as it
11348/// always did rather than refusing on a guess.
11349async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11350    let base = &state.base_branch;
11351    let mut against = base.clone();
11352    if mode == MergeMode::Pr {
11353        // A pull request lands on the remote's base, never the local branch
11354        // of the same name: if that cannot be read, "not empty" is the safe
11355        // answer.
11356        let remote = &state.config.merge.remote;
11357        let tracking = format!("{remote}/{base}");
11358        let fetched = git::fetch(repo, remote, base).await;
11359        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11360            against = tracking;
11361        } else {
11362            return false;
11363        }
11364    }
11365    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11366}
11367
11368/// Why nothing was opened for an empty winner, with what the task's own
11369/// references resolved to.
11370fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11371    let mut detail = format!(
11372        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11373         nothing to open a pull request for"
11374    );
11375    match refs::describe(&state.seeds) {
11376        Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11377        None => detail.push_str(
11378            "\nThe task names no existing branch or commit; if it means to land work \
11379             that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11380        ),
11381    }
11382    detail
11383}
11384
11385/// What the `Pr` merge does once it knows whether the branch already has an
11386/// open pull request.
11387#[derive(Debug, PartialEq, Eq)]
11388enum PrPlan {
11389    Create,
11390    Adopt { url: String, title: String },
11391    Stop(String),
11392}
11393
11394/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11395/// a failed lookup -> stop with the real reason. Never guesses.
11396fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11397    match found {
11398        Ok(land::OpenPr::None) => PrPlan::Create,
11399        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11400        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11401            "several open pull requests exist for this branch, not picking one: {}",
11402            urls.join(" ")
11403        )),
11404        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11405    }
11406}
11407
11408/// `gh pr create`, returning the PR url.
11409pub(crate) async fn gh_pr_create(
11410    cwd: &Path,
11411    base: &str,
11412    head: &str,
11413    title: &str,
11414    body: &str,
11415) -> Result<String> {
11416    let out = tokio::process::Command::new("gh")
11417        .args([
11418            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11419        ])
11420        .current_dir(cwd)
11421        .quiet()
11422        .stdin(std::process::Stdio::null())
11423        .output()
11424        .await
11425        .context("spawn gh")?;
11426    if out.status.success() {
11427        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11428    } else {
11429        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11430    }
11431}
11432
11433/// Never force deletion of uncommitted work, including unknown legacy trees.
11434async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11435    match std::fs::symlink_metadata(path) {
11436        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11437            // A previous cleanup may have removed the tree before a branch
11438            // deletion or state save failed. Unregister it if still present.
11439            git::worktree_remove_clean(repo, path).await?;
11440            return Ok(());
11441        }
11442        Err(e) => return Err(e.into()),
11443        Ok(_) => {}
11444    }
11445    if !git::is_clean(path).await? {
11446        bail!(
11447            "worktree has uncommitted work; refusing removal: {}",
11448            path.display()
11449        );
11450    }
11451    if !git::worktree_remove_clean(repo, path).await? {
11452        bail!("worktree removal refused: {}", path.display());
11453    }
11454    Ok(())
11455}
11456
11457/// Tear a run's worktrees and branches down.
11458///
11459/// `home` is where the updated `run.json` is saved (via
11460/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11461/// a housekeeping pass already has its own honest `home` handed to it, and
11462/// falling through to the global here would write back through whichever
11463/// directory some other process or test pinned into that `OnceLock` first,
11464/// not the one the caller actually resolved its `runs` and `state` from.
11465pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11466    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11467        bail!("refusing fold: {reason}");
11468    }
11469    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11470        bail!("candidate preservation incomplete; refusing cleanup");
11471    }
11472    // Serialize ordinary folding with the same starts/resumes as target-only
11473    // eviction. Otherwise a second janitor could remove the whole worktree
11474    // around a live output borrower even though target eviction was fenced.
11475    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11476    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11477    let _driver = crate::driver::maintenance(home, &state.id)?;
11478    let mut output_guards = Vec::new();
11479    for entry in std::fs::read_dir(state.worktree_root())
11480        .into_iter()
11481        .flatten()
11482        .flatten()
11483    {
11484        if entry.path().join(".git").is_file() {
11485            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11486        }
11487    }
11488    let repo = state.repo.clone();
11489    // CLI, Web and automatic cleanup all enter here, independently of
11490    // Runner::fold_losers. Protect deletion and shared config release too.
11491    let wait = repository_wait(state, None);
11492    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11493        Ok(lease) => lease,
11494        Err(e) => {
11495            state.save_under(home)?;
11496            return Err(e);
11497        }
11498    };
11499    // A merged PR proves only its published change. Preflight every source
11500    // before removing anything; a losing candidate or detached reviewer can
11501    // still retain independent commits or Git recovery checkpoints.
11502    let _source_outputs = if state.status == RunStatus::Merged
11503        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11504    {
11505        verify_merged_fold_sources(state, drop_winner, home).await?
11506    } else {
11507        Vec::new()
11508    };
11509    let root = state.worktree_root();
11510    let winner = state.tally.as_ref().map(|t| t.winner);
11511    let mut removed = Vec::new();
11512
11513    for i in 0..state.candidates.len() {
11514        let c = state.candidates[i].clone();
11515        let is_winner = Some(c.label) == winner;
11516        if is_winner && !drop_winner {
11517            continue;
11518        }
11519        let existed = c.worktree.exists();
11520        // A vanished directory can still have a Git registration that keeps
11521        // its branch checked out. Remove that registration before the branch.
11522        remove_preserved_worktree(&repo, &c.worktree).await?;
11523        if existed {
11524            removed.push(c.worktree.to_string_lossy().into_owned());
11525        }
11526        // A branch handed to a later run (and its pull request) is not this
11527        // run's to delete.
11528        let handed_over = state.released_branches.contains(&c.branch);
11529        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11530            if !git::branch_delete(&repo, &c.branch).await? {
11531                bail!("candidate branch removal refused");
11532            }
11533            removed.push(c.branch.clone());
11534        }
11535        state.candidates[i].folded = true;
11536    }
11537
11538    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11539        let path = name.path();
11540        let keep = !drop_winner
11541            && winner.is_some_and(|w| {
11542                path.file_name()
11543                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11544            });
11545        if keep {
11546            continue;
11547        }
11548        remove_preserved_worktree(&repo, &path).await?;
11549        removed.push(path.to_string_lossy().into_owned());
11550    }
11551
11552    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11553    // judge worktrees, so once the loop above has cleared all of them out,
11554    // the parent is a bare directory nobody else was ever going to remove -
11555    // git only ever managed what was inside it. Left alone, one of these
11556    // accumulates per fully-folded run; the operator's own machine had 74.
11557    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11558    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11559    // so does anything a run never claimed that happens to share the bay.
11560    remove_if_empty(&root);
11561
11562    if state.enabled_worktree_config && drop_winner {
11563        // A release, not a raw disable: some sibling run in this repository
11564        // may still hold its own reference (see `git::acquire_worktree_config`),
11565        // and only the last release actually turns the setting back off.
11566        git::release_worktree_config(&repo).await.ok();
11567        state.enabled_worktree_config = false;
11568    }
11569    state.save_under(home)?;
11570    Ok(removed)
11571}
11572
11573async fn verify_merged_fold_sources(
11574    state: &RunState,
11575    drop_winner: bool,
11576    home: &Path,
11577) -> Result<Vec<std::fs::File>> {
11578    let winner = state.tally.as_ref().map(|t| t.winner);
11579    let mut heads = BTreeSet::new();
11580    for candidate in &state.candidates {
11581        if Some(candidate.label) == winner && !drop_winner {
11582            continue;
11583        }
11584        match std::fs::symlink_metadata(&candidate.worktree) {
11585            Ok(_) => anyhow::ensure!(
11586                candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11587                "candidate recovery source is outside the inventoried run root: {}",
11588                candidate.worktree.display()
11589            ),
11590            Err(error) if error.kind() == std::io::ErrorKind::NotFound && candidate.folded => {}
11591            Err(error) => {
11592                return Err(error).context(format!(
11593                    "cannot inspect candidate recovery source: {}",
11594                    candidate.worktree.display()
11595                ));
11596            }
11597        }
11598        if git::branch_exists(&state.repo, &candidate.branch).await? {
11599            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11600            heads.extend(
11601                git::git(
11602                    &state.repo,
11603                    &["reflog", "show", "--format=%H", &candidate.branch],
11604                )
11605                .await?
11606                .lines()
11607                .map(str::to_owned),
11608            );
11609        }
11610    }
11611    let mut cargo_guards = Vec::new();
11612    let entries = match std::fs::read_dir(state.worktree_root()) {
11613        Ok(entries) => Some(entries),
11614        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11615        Err(error) => return Err(error.into()),
11616    };
11617    for entry in entries.into_iter().flatten() {
11618        let path = entry?.path();
11619        let metadata = std::fs::symlink_metadata(&path)?;
11620        anyhow::ensure!(
11621            metadata.is_dir() && !metadata.file_type().is_symlink(),
11622            "non-directory or symlink recovery source retained: {}",
11623            path.display()
11624        );
11625        if !drop_winner
11626            && winner.is_some_and(|w| {
11627                path.file_name()
11628                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11629            })
11630        {
11631            continue;
11632        }
11633        anyhow::ensure!(
11634            path.join(".git").is_file(),
11635            "unregistered recovery source retained: {}",
11636            path.display()
11637        );
11638        anyhow::ensure!(
11639            git::git(
11640                &path,
11641                &[
11642                    "status",
11643                    "--porcelain",
11644                    "--untracked-files=all",
11645                    "--ignored",
11646                    "--",
11647                    ".",
11648                    ":(exclude)target"
11649                ]
11650            )
11651            .await?
11652            .is_empty(),
11653            "source changes or ignored recovery files retained: {}",
11654            path.display()
11655        );
11656        match std::fs::symlink_metadata(path.join("target")) {
11657            Ok(_) => {
11658                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11659            }
11660            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11661            Err(error) => return Err(error.into()),
11662        }
11663        for checkpoint in [
11664            "rebase-merge",
11665            "rebase-apply",
11666            "sequencer",
11667            "CHERRY_PICK_HEAD",
11668            "MERGE_HEAD",
11669            "REBASE_HEAD",
11670            "REVERT_HEAD",
11671            "BISECT_LOG",
11672            "AUTO_MERGE",
11673            "MERGE_MSG",
11674        ] {
11675            let location = git::git(
11676                &path,
11677                &[
11678                    "rev-parse",
11679                    "--path-format=absolute",
11680                    "--git-path",
11681                    checkpoint,
11682                ],
11683            )
11684            .await?;
11685            anyhow::ensure!(
11686                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11687                if e.kind() == std::io::ErrorKind::NotFound),
11688                "Git recovery checkpoint retained: {checkpoint}"
11689            );
11690        }
11691        heads.insert(git::rev_parse(&path, "HEAD").await?);
11692        heads.extend(
11693            git::git(&path, &["reflog", "show", "--format=%H", "HEAD"])
11694                .await?
11695                .lines()
11696                .map(str::to_owned),
11697        );
11698        if let Ok(original) = git::rev_parse(&path, "ORIG_HEAD").await {
11699            heads.insert(original);
11700        }
11701    }
11702    if !heads.is_empty() {
11703        // Never trust a run-wide merge label or the local checkout's branch.
11704        // Pin the fetched base once. A squash does not preserve commit IDs:
11705        // prove the winner's cumulative patch before accepting its ancestors.
11706        let reference = format!(
11707            "refs/remotes/{}/{}",
11708            state.config.merge.remote, state.base_branch
11709        );
11710        let base = git::rev_parse(&state.repo, &reference)
11711            .await
11712            .context("cannot prove merged sources against the fetched base")?;
11713        let mut squash_winner = None;
11714        let mut checked_squash = false;
11715        for head in heads {
11716            if git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, &base])
11717                .await?
11718                .ok()
11719            {
11720                continue;
11721            }
11722            if !checked_squash {
11723                squash_winner = proven_squash_winner(state, &base).await?;
11724                checked_squash = true;
11725            }
11726            let incorporated = if let Some(winner) = &squash_winner {
11727                git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, winner])
11728                    .await?
11729                    .ok()
11730            } else {
11731                false
11732            };
11733            anyhow::ensure!(
11734                incorporated,
11735                "recovery source {head} is not proven incorporated in {reference}; retaining the entire run"
11736            );
11737        }
11738    }
11739    Ok(cargo_guards)
11740}
11741
11742/// Called only for a confirmed merged run. Its label admits proof, but does
11743/// not prove that any particular local branch or recovery checkpoint landed.
11744/// Match the complete binary patch, including modes and blob IDs, against a
11745/// single-parent commit in the pinned fetched history. This works after later
11746/// base commits too, without trusting the current base tree to still match.
11747async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
11748    let Some(winner) = state.winner() else {
11749        return Ok(None);
11750    };
11751    let head = git::rev_parse(&state.repo, &winner.branch).await?;
11752    if !git::git_raw(
11753        &state.repo,
11754        &["merge-base", "--is-ancestor", &state.base_commit, &head],
11755    )
11756    .await?
11757    .ok()
11758    {
11759        return Ok(None);
11760    }
11761    let patch = fold_source_patch(&state.repo, &state.base_commit, &head).await?;
11762    if patch.is_empty() {
11763        return Ok(None);
11764    }
11765    let range = format!("{}..{base}", state.base_commit);
11766    let commits = git::git(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
11767    for commit in commits.lines() {
11768        let parent = format!("{commit}^");
11769        if fold_source_patch(&state.repo, &parent, commit).await? == patch {
11770            return Ok(Some(head));
11771        }
11772    }
11773    Ok(None)
11774}
11775
11776async fn fold_source_patch(repo: &Path, from: &str, to: &str) -> Result<String> {
11777    git::git(
11778        repo,
11779        &[
11780            "-c",
11781            "core.quotePath=true",
11782            "diff",
11783            "--binary",
11784            "--full-index",
11785            "--no-renames",
11786            "--no-ext-diff",
11787            "--no-textconv",
11788            "--no-color",
11789            "--ignore-submodules=none",
11790            "--submodule=short",
11791            from,
11792            to,
11793            "--",
11794        ],
11795    )
11796    .await
11797}
11798
11799/// Remove `dir` if it exists and has nothing in it.
11800///
11801/// Best-effort and silent by design: a directory that is not empty (a run
11802/// whose winner is still parked there, a stray file some other process left)
11803/// is exactly the case this must refuse, and a directory that is already gone
11804/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11805/// already refuses a non-empty directory, so the emptiness check below is
11806/// belt, not suspenders - it is what keeps this from ever attempting the
11807/// removal in the case that matters, rather than trusting `remove_dir`'s
11808/// error path to have no side effects if it ever changed.
11809fn remove_if_empty(dir: &Path) {
11810    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11811        std::fs::remove_dir(dir).ok();
11812    }
11813}
11814
11815/// Severity of the worst open finding in the last review round, for reporting.
11816pub fn worst_open(state: &RunState) -> Option<Severity> {
11817    state
11818        .reviews
11819        .last()?
11820        .reviews
11821        .iter()
11822        .flat_map(|r| r.findings.iter())
11823        .map(|f| f.severity)
11824        .max()
11825}
11826
11827/// `[worktree] setup` hides its products per worktree, which needs
11828/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11829/// worktrees, like the hook's; every place that runs setup calls this first.
11830async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11831    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11832        git::acquire_worktree_config(repo).await?;
11833        state.enabled_worktree_config = true;
11834    }
11835    Ok(())
11836}
11837
11838#[cfg(test)]
11839mod tests {
11840    #[tokio::test]
11841    async fn launcher_base_fetch_requires_remote_success() {
11842        let tmp = tempfile::tempdir().unwrap();
11843        let repo = tmp.path().join("repo");
11844        let remote = tmp.path().join("remote");
11845        std::fs::create_dir(&repo).unwrap();
11846        std::fs::create_dir(&remote).unwrap();
11847        git::git(&remote, &["init", "--bare", "-b", "main"])
11848            .await
11849            .unwrap();
11850        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11851        git::git(
11852            &repo,
11853            &[
11854                "-c",
11855                "user.name=Fixture",
11856                "-c",
11857                "user.email=fixture@example.invalid",
11858                "commit",
11859                "--allow-empty",
11860                "-m",
11861                "fixture",
11862            ],
11863        )
11864        .await
11865        .unwrap();
11866        git::git(
11867            &repo,
11868            &["remote", "add", "origin", remote.to_str().unwrap()],
11869        )
11870        .await
11871        .unwrap();
11872        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11873        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11874        assert_eq!(
11875            resolve_base(&repo, "main", "origin", None)
11876                .await
11877                .unwrap()
11878                .commit,
11879            head
11880        );
11881        #[cfg(unix)]
11882        {
11883            let policy: crate::config::Coordination = toml::from_str(
11884                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11885            ).unwrap();
11886            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11887            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11888            let held = crate::repo_lock::SeatLease::operation_with(
11889                &repo,
11890                Some(&policy),
11891                &crate::repo_lock::WaitPolicy::default(),
11892            )
11893            .await
11894            .unwrap();
11895            assert!(held.is_some());
11896            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11897            tokio::pin!(fetch);
11898            assert!(
11899                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11900                    .await
11901                    .is_err()
11902            );
11903            assert!(
11904                !repo.join(".git/FETCH_HEAD").exists(),
11905                "no fetch before admission"
11906            );
11907            drop(held);
11908            let fetched = fetch.await.unwrap();
11909            assert_eq!(fetched.commit, head);
11910            assert_eq!(fetched.remote, "origin");
11911            assert_eq!(fetched.branch, "main");
11912        }
11913        // A surviving tracking ref is not successful fetch evidence.
11914        git::git(
11915            &repo,
11916            &[
11917                "remote",
11918                "set-url",
11919                "origin",
11920                tmp.path().join("missing").to_str().unwrap(),
11921            ],
11922        )
11923        .await
11924        .unwrap();
11925        let error = resolve_base(&repo, "main", "origin", None)
11926            .await
11927            .unwrap_err()
11928            .to_string();
11929        assert!(error.contains("cannot read origin/main"));
11930        assert!(error.contains("refusing to branch off"));
11931    }
11932
11933    #[test]
11934    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11935        let leaked = vec!["chore(deps): update a crate".to_owned()];
11936        let own = "fix(daemon): apply a chosen action";
11937        assert!(should_retitle("", own, &leaked));
11938        assert!(should_retitle(
11939            &format!("{REVIEW_PROMPT_OPENING} `x`"),
11940            own,
11941            &leaked
11942        ));
11943        assert!(should_retitle(
11944            "chore: land candidate A of run 1",
11945            own,
11946            &leaked
11947        ));
11948        assert!(should_retitle(
11949            "magi: candidate A (uncommitted work)",
11950            own,
11951            &leaked
11952        ));
11953        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11954        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11955        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11956    }
11957
11958    #[test]
11959    fn pr_merge_plan_creates_adopts_or_stops() {
11960        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11961        assert_eq!(
11962            pr_merge_plan(Ok(land::OpenPr::One {
11963                url: "u".into(),
11964                title: "t".into()
11965            })),
11966            PrPlan::Adopt {
11967                url: "u".into(),
11968                title: "t".into()
11969            }
11970        );
11971        let PrPlan::Stop(many) =
11972            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11973        else {
11974            panic!("many must stop");
11975        };
11976        assert!(many.contains('a') && many.contains('b'));
11977        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11978            panic!("a failed lookup must stop");
11979        };
11980        assert!(err.contains("bad token"));
11981    }
11982
11983    use super::*;
11984    use crate::run::GateStatus;
11985    use std::collections::BTreeMap;
11986    use std::time::Duration;
11987
11988    fn conductor() -> AgentSpec {
11989        AgentSpec {
11990            id: "conductor".to_owned(),
11991            kind: crate::config::AgentKind::Command,
11992            model: None,
11993            command: vec!["true".to_owned()],
11994            extra_args: Vec::new(),
11995            env: BTreeMap::new(),
11996            prompt_delivery: None,
11997        }
11998    }
11999
12000    fn spec(id: &str) -> AgentSpec {
12001        AgentSpec {
12002            id: id.to_owned(),
12003            kind: crate::config::AgentKind::Command,
12004            model: None,
12005            command: vec!["true".to_owned()],
12006            extra_args: Vec::new(),
12007            env: BTreeMap::new(),
12008            prompt_delivery: None,
12009        }
12010    }
12011
12012    fn ids(xs: &[&str]) -> BTreeSet<String> {
12013        xs.iter().map(|s| (*s).to_owned()).collect()
12014    }
12015
12016    #[test]
12017    fn pick_successor_skips_an_agent_another_seat_holds() {
12018        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12019        let roster = [spec("a"), spec("b"), spec("c")];
12020        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12021        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12022    }
12023
12024    #[test]
12025    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12026        // The other seat started on c but was handed to d; c is free again.
12027        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12028        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12029        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12030        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12031        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12032    }
12033
12034    #[test]
12035    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12036        let roster = [spec("a"), spec("b")];
12037        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12038        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12039        let carried = ids(&["b"]);
12040        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12041        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12042    }
12043
12044    #[test]
12045    fn pick_successor_returns_none_without_any_untried_successor() {
12046        let roster = [spec("a"), spec("b")];
12047        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12048        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12049        let carried = ids(&["a"]);
12050        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12051    }
12052
12053    #[test]
12054    fn pick_successor_rescue_avoids_another_seats_occupant() {
12055        // b is a carried failure and free; a is held by the other seat.
12056        let roster = [spec("a"), spec("b"), spec("c")];
12057        let carried = ids(&["b", "c"]);
12058        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12059        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12060    }
12061
12062    #[test]
12063    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12064        let roster = [spec("a"), spec("b"), spec("c")];
12065        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12066        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12067    }
12068
12069    #[test]
12070    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12071        let roster = [spec("a"), spec("b"), spec("c")];
12072        let failed = ids(&["a", "b", "c"]);
12073        // Rescue looks at the whole roster, once per id, then runs out.
12074        let mut tried = ids(&["b"]);
12075        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12076        assert_eq!(first.id, "a");
12077        tried.insert(first.id.clone());
12078        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12079        assert_eq!(second.id, "c");
12080        tried.insert(second.id.clone());
12081        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12082    }
12083
12084    #[test]
12085    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12086        let roster = [spec("a"), spec("b")];
12087        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12088        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12089    }
12090
12091    #[test]
12092    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12093        let roster = [spec("a"), spec("b"), spec("c")];
12094        let h = SeatHistory {
12095            failed: ids(&["a"]),
12096            last_ok: Some("b".to_owned()),
12097            last_fail: None,
12098        };
12099        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12100        // A last answerer that left the roster, or later failed, is ignored.
12101        let gone = SeatHistory {
12102            last_ok: Some("zzz".to_owned()),
12103            ..h.clone()
12104        };
12105        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12106        let failed = SeatHistory {
12107            failed: ids(&["a", "b"]),
12108            last_ok: Some("b".to_owned()),
12109            last_fail: None,
12110        };
12111        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12112    }
12113
12114    #[test]
12115    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12116        let first = SeatState::new("review-1", "alpha", 7);
12117        let next = handover_seat("review-1", "gamma", 7);
12118        assert_ne!(first.claude_session, next.claude_session);
12119    }
12120
12121    #[test]
12122    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12123        let mut state = state_with_summary("x", "y");
12124        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12125        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12126        assert_ne!(a.claude_session, b.claude_session);
12127    }
12128
12129    #[test]
12130    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12131        let roster = [spec("a"), spec("b")];
12132        let h = SeatHistory {
12133            failed: ids(&["a", "b"]),
12134            ..SeatHistory::default()
12135        };
12136        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12137        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12138        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12139    }
12140
12141    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12142    // fallback loop depends on to terminate: it must walk forward from the
12143    // seat's own position, never restart at the front of the roster, and it
12144    // must never hand back an id already tried, however many times that id
12145    // happens to appear.
12146
12147    #[test]
12148    fn failure_signature_ignores_numbers_and_paths() {
12149        assert_eq!(
12150            failure_signature("exited with Some(2) and no usable output"),
12151            failure_signature("exited with Some(137) and no usable output")
12152        );
12153        assert_eq!(
12154            failure_signature("process exited with no usable output (exit code 2)"),
12155            failure_signature("process exited with no usable output (exit code 137)")
12156        );
12157        for (old, new) in [
12158            (
12159                "exited with Some(2) and no usable output",
12160                "process exited with no usable output (exit code 2)",
12161            ),
12162            (
12163                "exited with None and no usable output",
12164                "process exited with no usable output (exit code unavailable)",
12165            ),
12166        ] {
12167            let previous = FailClass::Other(failure_signature(old));
12168            let current = FailClass::Other(failure_signature(new));
12169            assert_eq!(previous, current, "resumption retains the failure class");
12170            assert!(!should_hand_over(Some(&previous), &current));
12171        }
12172        assert_eq!(
12173            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12174            failure_signature("cannot open /var/x.txt: denied")
12175        );
12176        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12177    }
12178
12179    #[test]
12180    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12181        let other = FailClass::Other("x".into());
12182        assert!(should_hand_over(None, &FailClass::Quota));
12183        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12184        assert!(should_hand_over(
12185            Some(&FailClass::Timeout),
12186            &FailClass::Timeout
12187        ));
12188        assert!(should_hand_over(None, &other));
12189        assert!(!should_hand_over(Some(&other), &other));
12190        assert!(should_hand_over(
12191            Some(&other),
12192            &FailClass::Other("y".into())
12193        ));
12194        // A quota or timeout in between ends the run of identical failures.
12195        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12196        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12197    }
12198
12199    #[test]
12200    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12201        let a = SeatState::new("judge-1", "alpha", 7);
12202        let b = handover_seat("judge-1", "beta", 7);
12203        assert_ne!(a.claude_session, b.claude_session);
12204        assert_eq!(b.turns, 0);
12205    }
12206
12207    #[test]
12208    fn a_timeout_is_classified_apart_from_other_failures() {
12209        assert_eq!(
12210            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12211            Some(FailClass::Timeout)
12212        );
12213        assert!(matches!(
12214            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12215            Some(FailClass::Other(_))
12216        ));
12217    }
12218
12219    #[test]
12220    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12221        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12222        let tried = BTreeSet::from(["beta".to_owned()]);
12223        // beta sits at index 1; the next candidate is gamma, never alpha —
12224        // which is very likely a different candidate slot's own agent.
12225        let next = next_untried_in_roster(&roster, 1, &tried);
12226        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12227    }
12228
12229    #[test]
12230    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12231        let roster = vec![spec("alpha"), spec("beta")];
12232        let tried = BTreeSet::from(["beta".to_owned()]);
12233        // beta is the roster's last entry: nothing follows it, and alpha —
12234        // earlier in the roster, almost certainly a different candidate
12235        // slot's own agent — must not be reached by wrapping back to it.
12236        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12237    }
12238
12239    #[test]
12240    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12241        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12242        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12243        // beta (index 1) and gamma (index 2, the only entry after it) have
12244        // both been tried; alpha (index 0) never has, but it comes before
12245        // beta's own position, so there is nothing further for this seat.
12246        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12247    }
12248
12249    #[test]
12250    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12251        let roster = vec![spec("a"), spec("a"), spec("b")];
12252        let tried = BTreeSet::from(["a".to_owned()]);
12253        let next = next_untried_in_roster(&roster, 0, &tried);
12254        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12255    }
12256
12257    #[test]
12258    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12259        let roster = vec![spec("a"), spec("b")];
12260        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12261        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12262    }
12263
12264    #[test]
12265    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12266        let dir = tempfile::tempdir().unwrap();
12267        let bay = dir.path().join("ffff");
12268
12269        // Not there yet: nothing to do, nothing to panic on.
12270        remove_if_empty(&bay);
12271        assert!(!bay.exists());
12272
12273        // Something still inside - the winner's worktree, or a stray file -
12274        // keeps the directory standing.
12275        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12276        remove_if_empty(&bay);
12277        assert!(bay.exists(), "non-empty directory must survive");
12278
12279        // Once the last entry is gone, so is the directory itself.
12280        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12281        remove_if_empty(&bay);
12282        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12283    }
12284
12285    // `round_is_clean` is the exact decision this task fixed: a round with a
12286    // seat that never answered must not read the same as a round every seat
12287    // actually reviewed. These are deterministic and process-free by design —
12288    // the equivalent end-to-end check (a real reviewer timing out under a
12289    // live graph run) is a genuine race against wall-clock contention, and a
12290    // spawn slow enough to blow even a generous budget under a loaded test
12291    // run must not turn this specific regression check flaky.
12292
12293    #[test]
12294    fn a_full_panel_that_found_nothing_is_clean() {
12295        assert!(round_is_clean(
12296            0,
12297            true,
12298            2,
12299            2,
12300            0,
12301            IncompleteReviewPolicy::Block
12302        ));
12303    }
12304
12305    #[test]
12306    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12307        assert!(!round_is_clean(
12308            0,
12309            true,
12310            1,
12311            2,
12312            0,
12313            IncompleteReviewPolicy::Block
12314        ));
12315    }
12316
12317    #[test]
12318    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12319        assert!(!round_is_clean(
12320            1,
12321            true,
12322            1,
12323            2,
12324            0,
12325            IncompleteReviewPolicy::Warn
12326        ));
12327    }
12328
12329    #[test]
12330    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12331        assert!(round_is_clean(
12332            0,
12333            true,
12334            1,
12335            2,
12336            0,
12337            IncompleteReviewPolicy::Warn
12338        ));
12339    }
12340
12341    #[test]
12342    fn a_full_panel_with_an_open_finding_is_not_clean() {
12343        assert!(!round_is_clean(
12344            1,
12345            true,
12346            2,
12347            2,
12348            0,
12349            IncompleteReviewPolicy::Block
12350        ));
12351    }
12352
12353    #[test]
12354    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12355        assert!(!round_is_clean(
12356            0,
12357            false,
12358            2,
12359            2,
12360            0,
12361            IncompleteReviewPolicy::Block
12362        ));
12363    }
12364
12365    // The stall this task closes: under the default `block` policy, a seat
12366    // missing only because it was rate limited must not force a wait for a
12367    // session limit that will not lift by the next round. `round_is_clean`
12368    // is where that quorum carve-out lives; the review loop around it never
12369    // changes what a reviewer's vote or a finding's severity means.
12370
12371    #[test]
12372    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12373        // 1 of 2 answered, and the one missing was quota'd — the exact
12374        // "review-2 rate limited (quota)" shape from the field report.
12375        assert!(round_is_clean(
12376            0,
12377            true,
12378            1,
12379            2,
12380            1,
12381            IncompleteReviewPolicy::Block
12382        ));
12383    }
12384
12385    #[test]
12386    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12387        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12388        // not a quota loss (`quota_missing` stays 0) — worth another try.
12389        assert!(!round_is_clean(
12390            0,
12391            true,
12392            1,
12393            2,
12394            0,
12395            IncompleteReviewPolicy::Block
12396        ));
12397    }
12398
12399    #[test]
12400    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12401        assert!(!round_is_clean(
12402            1,
12403            true,
12404            1,
12405            2,
12406            1,
12407            IncompleteReviewPolicy::Block
12408        ));
12409        assert!(!round_is_clean(
12410            0,
12411            false,
12412            1,
12413            2,
12414            1,
12415            IncompleteReviewPolicy::Block
12416        ));
12417    }
12418
12419    #[test]
12420    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12421        // Every seat quota'd, nobody answered: there is no panel to decide
12422        // on, so this must fall through to the existing block-and-retry
12423        // fallback rather than call an unreviewed patch clean.
12424        assert!(!round_is_clean(
12425            0,
12426            true,
12427            0,
12428            2,
12429            2,
12430            IncompleteReviewPolicy::Block
12431        ));
12432    }
12433
12434    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12435        CommandOutcome {
12436            command: "test".to_owned(),
12437            code,
12438            output_tail: String::new(),
12439            duration_ms: 0,
12440            resource_blocked,
12441        }
12442    }
12443
12444    #[test]
12445    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12446        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12447        assert!(
12448            !verify_inconclusive(&[outcome(Some(1), false)]),
12449            "an ordinary failure is still evidence about the patch"
12450        );
12451        assert!(verify_inconclusive(&[outcome(None, true)]));
12452        assert!(
12453            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12454            "one inconclusive outcome taints the whole batch"
12455        );
12456        assert!(!verify_inconclusive(&[]));
12457    }
12458
12459    #[tokio::test]
12460    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12461        // Alive for the first two checks, then dead - confirms the loop
12462        // actually re-polls rather than deciding once and sleeping out the
12463        // ceiling regardless.
12464        let calls = std::sync::atomic::AtomicUsize::new(0);
12465        let started = Instant::now();
12466        wait_for_pids_with(
12467            &[123],
12468            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12469            Duration::from_millis(5),
12470            Duration::from_secs(5),
12471        )
12472        .await;
12473        assert!(
12474            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12475            "must keep checking rather than deciding on the first answer"
12476        );
12477        assert!(
12478            started.elapsed() < Duration::from_secs(1),
12479            "must return the moment it is confirmed dead, not wait out the ceiling"
12480        );
12481    }
12482
12483    #[tokio::test]
12484    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12485        let started = Instant::now();
12486        wait_for_pids_with(
12487            &[123],
12488            |_| true, // never reports dead
12489            Duration::from_millis(5),
12490            Duration::from_millis(30),
12491        )
12492        .await;
12493        let elapsed = started.elapsed();
12494        assert!(
12495            elapsed >= Duration::from_millis(30),
12496            "must not give up before its own ceiling: {elapsed:?}"
12497        );
12498        assert!(
12499            elapsed < Duration::from_secs(1),
12500            "must not wait past its own ceiling either: {elapsed:?}"
12501        );
12502    }
12503
12504    #[tokio::test]
12505    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12506        let started = Instant::now();
12507        wait_for_pids_with(
12508            &[],
12509            |_| true,
12510            Duration::from_secs(5),
12511            Duration::from_secs(5),
12512        )
12513        .await;
12514        assert!(
12515            started.elapsed() < Duration::from_millis(200),
12516            "an empty pid list has nothing to confirm"
12517        );
12518    }
12519
12520    // `review_conclusion` is the exact decision the review hand-off task
12521    // fixed: a round budget spent (or a tree that stopped moving) must not
12522    // collapse into `Blocked` regardless of what verification actually
12523    // said. Deterministic and process-free for the same reason the
12524    // `round_is_clean` family above is.
12525    fn review_round(
12526        clean: bool,
12527        blocking: usize,
12528        answered: usize,
12529        expected: usize,
12530        progressed: bool,
12531        e2e_ok: bool,
12532    ) -> ReviewRound {
12533        ReviewRound {
12534            round: 1,
12535            head: "h".to_owned(),
12536            verified_head: None,
12537            verified_at: None,
12538            reviews: Vec::new(),
12539            e2e: vec![CommandOutcome {
12540                command: "test".to_owned(),
12541                code: Some(if e2e_ok { 0 } else { 1 }),
12542                output_tail: String::new(),
12543                duration_ms: 0,
12544                resource_blocked: false,
12545            }],
12546            verify_retried: false,
12547            e2e_deferred: false,
12548            e2e_defer_reason: None,
12549            fix: None,
12550            blocking,
12551            answered,
12552            expected,
12553            clean,
12554            progressed,
12555            vote_split: false,
12556            reconsideration: Vec::new(),
12557            verdict: None,
12558        }
12559    }
12560
12561    #[test]
12562    fn review_conclusion_is_none_when_nothing_has_run() {
12563        assert_eq!(review_conclusion(&[], 3), None);
12564    }
12565
12566    #[test]
12567    fn review_conclusion_is_none_while_rounds_remain() {
12568        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12569        assert_eq!(review_conclusion(&rounds, 3), None);
12570    }
12571
12572    #[test]
12573    fn review_conclusion_is_gating_once_a_round_is_clean() {
12574        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12575        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12576    }
12577
12578    #[test]
12579    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12580        let rounds = vec![
12581            review_round(false, 1, 2, 2, true, true),
12582            review_round(false, 1, 2, 2, true, true),
12583        ];
12584        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12585    }
12586
12587    #[test]
12588    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12589        let rounds = vec![
12590            review_round(false, 1, 2, 2, true, true),
12591            review_round(false, 1, 2, 2, true, false),
12592        ];
12593        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12594    }
12595
12596    #[test]
12597    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12598        // Magi never got a command to run against this round's own head — a
12599        // resource-blocked attempt, not a red one — so this must never
12600        // settle on `Blocked` the way a genuine e2e failure would. `None`
12601        // here is what tells `Runner::review_loop` to retry the check
12602        // itself rather than trust this cheap recomputation with a verdict
12603        // it cannot actually produce.
12604        let mut blocked = review_round(false, 1, 2, 2, true, false);
12605        blocked.e2e[0].resource_blocked = true;
12606        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12607        assert_eq!(review_conclusion(&rounds, 2), None);
12608    }
12609
12610    #[test]
12611    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12612        // Missing input, not a verified tree — never a hand-off candidate.
12613        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12614        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12615    }
12616
12617    #[test]
12618    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12619        let rounds = vec![
12620            review_round(false, 1, 2, 2, false, true),
12621            review_round(false, 1, 2, 2, false, true),
12622        ];
12623        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12624    }
12625
12626    fn secs(n: u64) -> Duration {
12627        Duration::from_secs(n)
12628    }
12629
12630    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12631    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12632    #[tokio::test]
12633    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12634        ask_test_home();
12635        let tmp = tempfile::tempdir().unwrap();
12636        let repo = tmp.path().join("repo");
12637        std::fs::create_dir_all(&repo).unwrap();
12638        init_repo(&repo);
12639        let g = |args: &[&str]| {
12640            let out = std::process::Command::new("git")
12641                .args(args)
12642                .current_dir(&repo)
12643                .quiet()
12644                .output()
12645                .unwrap();
12646            assert!(
12647                out.status.success(),
12648                "{}",
12649                String::from_utf8_lossy(&out.stderr)
12650            );
12651        };
12652        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12653        g(&["checkout", "-b", "historical"]);
12654        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12655        g(&["add", "."]);
12656        g(&["commit", "-m", "old alternative"]);
12657        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12658        g(&["checkout", "-b", "magi/task-selected", &base]);
12659        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12660        g(&["add", "."]);
12661        g(&["commit", "-m", "measurement infrastructure"]);
12662        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12663        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12664        g(&["add", "."]);
12665        g(&["commit", "-m", "dependent implementation"]);
12666        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12667        g(&["checkout", "main"]);
12668        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12669        g(&["add", "."]);
12670        g(&["commit", "-m", "unrelated base change"]);
12671        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12672        let origin = tmp.path().join("origin.git");
12673        g(&[
12674            "clone",
12675            "--bare",
12676            repo.to_str().unwrap(),
12677            origin.to_str().unwrap(),
12678        ]);
12679        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12680        g(&["fetch", "origin"]);
12681        let selection = crate::pr_recovery::Selection {
12682            url: "https://example.test/project/pull/1".into(),
12683            branch: "magi/task-selected".into(),
12684            head: head.clone(),
12685            source_head: None,
12686            source_run: None,
12687            source_proof: None,
12688            disposition: Some(crate::pr_recovery::Disposition::Review),
12689            instruction: "preserve measured behavior".into(),
12690            approved_at: jiff::Timestamp::now(),
12691        };
12692        let mut template = runner_at(RunStatus::Reviewing);
12693        template.state.repo = repo.clone();
12694        template.state.base_commit = current.clone();
12695        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12696        template.state.origin = Some(crate::run::Origin::queue("selected"));
12697        template.state.pr_head = Some(selection.branch.clone());
12698        template.state.pr_recovery = Some(selection.clone());
12699        let mut runner =
12700            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12701                .await
12702                .unwrap();
12703        let wt = runner.state.winner().unwrap().worktree.clone();
12704        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12705        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12706        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12707        assert!(runner.state.seeds.is_empty());
12708        assert_eq!(runner.state.instruction, "preserve measured behavior");
12709        runner.sync_to_base().await.unwrap();
12710        assert_eq!(
12711            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12712            "prerequisite"
12713        );
12714        assert_eq!(
12715            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12716            "measured implementation"
12717        );
12718        assert_eq!(
12719            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12720            "current main change"
12721        );
12722        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12723        assert_eq!(
12724            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12725            head
12726        );
12727
12728        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12729        let mut partial = runner_at(RunStatus::Reviewing);
12730        partial.state.repo = repo.clone();
12731        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12732        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12733            run: Some("exit 1".into()),
12734            ..Default::default()
12735        }];
12736        let selection = runner.state.pr_recovery.clone().unwrap();
12737        partial.state.pr_recovery = Some(selection.clone());
12738        partial.state.pr_head = Some(selection.branch.clone());
12739        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12740        let id = partial.state.id.clone();
12741        let error = match Runner::open_selected(
12742            &repo,
12743            partial.state,
12744            partial.roles,
12745            1,
12746            runner.state.base_commit.clone(),
12747            selection,
12748        )
12749        .await
12750        {
12751            Ok(_) => panic!("setup must fail"),
12752            Err(e) => e,
12753        };
12754        assert_eq!(
12755            error
12756                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12757                .unwrap()
12758                .run,
12759            id
12760        );
12761        let failed = RunState::load(&id).unwrap();
12762        assert_eq!(failed.status, RunStatus::Blocked);
12763        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12764        assert!(Runner::resume(&id).is_err());
12765        runner.state.status = RunStatus::Ready;
12766        runner.state.driver_exited = true;
12767        runner.state.save().unwrap();
12768        let mut retry = runner_at(RunStatus::Reviewing);
12769        retry.state.repo = repo.clone();
12770        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12771        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12772        reselected.approved_at = jiff::Timestamp::now();
12773        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12774        retry.state.pr_head = Some(reselected.branch.clone());
12775        retry.state.pr_recovery = Some(reselected.clone());
12776        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12777            .await
12778            .unwrap();
12779        let restarted = Runner::open_selected(
12780            &repo,
12781            retry.state,
12782            retry.roles,
12783            1,
12784            runner.state.base_commit.clone(),
12785            reselected,
12786        )
12787        .await
12788        .unwrap();
12789        assert_eq!(
12790            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12791                .await
12792                .unwrap(),
12793            head
12794        );
12795        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12796        assert_eq!(
12797            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12798            head
12799        );
12800    }
12801
12802    #[tokio::test]
12803    #[cfg(unix)]
12804    async fn fallback_and_interrupted_restoration_require_coordination() {
12805        use std::os::fd::AsRawFd as _;
12806        ask_test_home();
12807        let tmp = tempfile::tempdir().unwrap();
12808        let mut runner = registered_runner(tmp.path()).await;
12809        runner.state.config.graph.timeout_resource_wait = 1;
12810        let winner = runner.state.candidates[0].clone();
12811        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12812        let leftover = winner.worktree.join("leftover");
12813        std::fs::write(&leftover, "untrusted edits").unwrap();
12814        let next = tmp.path().join("next-fixer");
12815        let mut failed = spec("first");
12816        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12817        let mut successor = spec("second");
12818        successor.command = vec![
12819            "sh".into(),
12820            "-c".into(),
12821            format!("touch '{}'; echo done", next.display()),
12822        ];
12823        let attempts = vec![
12824            (failed.clone(), "first".into()),
12825            (successor.clone(), "second".into()),
12826        ];
12827        let lock_path = runner.state.repo.join(".git/updater.lock");
12828        let external = std::fs::File::create(&lock_path).unwrap();
12829        assert_eq!(
12830            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12831            0
12832        );
12833        let artifacts = tmp.path().join("artifacts");
12834        let build = |spec, seat| SeatJob {
12835            spec,
12836            seat,
12837            cwd: winner.worktree.clone(),
12838            prompt: "fixture".into(),
12839            timeout: Duration::from_secs(10),
12840            allow_write: true,
12841            sessions: false,
12842            artifacts: artifacts.clone(),
12843            stem: "fix".into(),
12844            handover: None,
12845        };
12846        let (_, _, out) = runner
12847            .ask_fixer_among(
12848                attempts.clone(),
12849                "graft",
12850                None,
12851                Some((&winner.worktree, &head)),
12852                build,
12853            )
12854            .await;
12855        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12856        assert!(leftover.exists(), "contention must prevent clean");
12857        assert!(!next.exists());
12858        assert!(runner.state.handovers.is_empty());
12859        // Early interrupted-repair rollback uses the same protected route.
12860        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12861            from_run: "earlier".into(),
12862            generation: 1,
12863            mode: crate::recovery::Mode::VerifyRepair,
12864            reasons: Vec::new(),
12865            reviewed_head: None,
12866            rounds: Vec::new(),
12867            open_findings: Vec::new(),
12868            evidence: None,
12869            criteria_inherited: true,
12870            repair: crate::recovery::Repair::Started {
12871                before: head.clone(),
12872            },
12873        });
12874        assert!(
12875            runner
12876                .recovery_repair(&winner)
12877                .await
12878                .unwrap_err()
12879                .is::<crate::repo_lock::ResourceBlocked>()
12880        );
12881        assert!(matches!(
12882            runner.state.recovery.as_ref().unwrap().repair,
12883            crate::recovery::Repair::Started { .. }
12884        ));
12885        assert!(leftover.exists());
12886        drop(external);
12887        runner.recovery_repair(&winner).await.unwrap();
12888        assert!(!leftover.exists());
12889        assert!(matches!(
12890            runner.state.recovery.as_ref().unwrap().repair,
12891            crate::recovery::Repair::Done { .. }
12892        ));
12893        // A hard lock-open failure also leaves HEAD and untracked work intact.
12894        std::fs::remove_file(&lock_path).unwrap();
12895        std::fs::create_dir(&lock_path).unwrap();
12896        std::fs::write(&leftover, "untrusted edits").unwrap();
12897        let (_, _, out) = runner
12898            .ask_fixer_among(
12899                attempts,
12900                "graft",
12901                None,
12902                Some((&winner.worktree, &head)),
12903                build,
12904            )
12905            .await;
12906        assert!(matches!(out, AgentOutcome::Failed(_)));
12907        assert!(leftover.exists());
12908        assert!(!next.exists());
12909        assert_eq!(
12910            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12911            head
12912        );
12913    }
12914
12915    #[cfg(unix)]
12916    async fn registered_runner(root: &Path) -> Runner {
12917        ask_test_home();
12918        let repo = root.join("repo");
12919        std::fs::create_dir_all(&repo).unwrap();
12920        init_repo(&repo);
12921        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12922        let mut runner = runner_at(RunStatus::Prep);
12923        runner.state.repo = repo.clone();
12924        runner.state.config.coordination = crate::config::test_coordination();
12925        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12926        runner.state.config.graph.worktree_root = Some(root.join("wt"));
12927        runner.state.config.graph.advise = false;
12928        runner.roles.implementers = vec![spec("implement")];
12929        runner.roles.judges = vec![spec("judge")];
12930        runner.prep().await.unwrap();
12931        runner
12932    }
12933
12934    #[tokio::test]
12935    #[cfg(unix)]
12936    async fn parked_implementer_persists_completed_work() {
12937        for registered in [false, true] {
12938            let tmp = tempfile::tempdir().unwrap();
12939            let mut runner = registered_runner(tmp.path()).await;
12940            if !registered {
12941                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12942            }
12943            let marker = tmp.path().join("started");
12944            let release = tmp.path().join("release");
12945            let mut implementer = spec("implement");
12946            implementer.command = vec![
12947                "sh".into(),
12948                "-c".into(),
12949                format!(
12950                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12951                    marker.display(),
12952                    release.display()
12953                ),
12954            ];
12955            runner.roles.implementers = vec![implementer.clone()];
12956            runner.state.config.agents = vec![implementer];
12957            let pause = runner.pause.clone();
12958            let (result, ()) = tokio::join!(runner.implement(), async {
12959                tokio::time::timeout(Duration::from_secs(10), async {
12960                    while !marker.exists() {
12961                        tokio::time::sleep(Duration::from_millis(10)).await;
12962                    }
12963                })
12964                .await
12965                .unwrap();
12966                pause.park();
12967                std::fs::write(&release, "go").unwrap();
12968            });
12969            result.unwrap();
12970            let candidate = &runner.state.candidates[0];
12971            assert!(candidate.commits > 0, "{candidate:?}");
12972            assert!(candidate.summary.contains("saved-result"));
12973            assert_eq!(candidate.files, 1);
12974            let before = candidate.commits;
12975            runner.implement().await.unwrap();
12976            assert_eq!(runner.state.candidates[0].commits, before);
12977        }
12978    }
12979
12980    #[tokio::test]
12981    #[cfg(unix)]
12982    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12983        let tmp = tempfile::tempdir().unwrap();
12984        let mut runner = registered_runner(tmp.path()).await;
12985        let winner = runner.state.candidates[0].clone();
12986        let cache = tmp.path().join("cache");
12987        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12988        runner.state.config.graph.timeout_resource_wait = 0;
12989        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12990        let crate::cache::AcquireOutcome::Acquired(held) =
12991            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12992        else {
12993            panic!("cache must be free")
12994        };
12995        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12996            from_run: "earlier".into(),
12997            generation: 1,
12998            mode: crate::recovery::Mode::VerifyRepair,
12999            reasons: Vec::new(),
13000            reviewed_head: None,
13001            rounds: Vec::new(),
13002            open_findings: Vec::new(),
13003            evidence: None,
13004            criteria_inherited: true,
13005            repair: crate::recovery::Repair::NotStarted,
13006        });
13007        runner.recovery_repair(&winner).await.unwrap();
13008        assert!(matches!(
13009            runner.state.recovery.as_ref().unwrap().repair,
13010            crate::recovery::Repair::WaitingForRestore { .. }
13011        ));
13012        drop(held);
13013        runner.state.resource_block = None;
13014        runner.state.config.graph.timeout_resource_wait = 1;
13015        runner.recovery_repair(&winner).await.unwrap();
13016        assert!(matches!(
13017            runner.state.recovery.as_ref().unwrap().repair,
13018            crate::recovery::Repair::Done { .. }
13019        ));
13020        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13021    }
13022
13023    #[tokio::test]
13024    #[cfg(unix)]
13025    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13026        use std::os::fd::AsRawFd as _;
13027        for graft in [false, true] {
13028            let tmp = tempfile::tempdir().unwrap();
13029            let mut runner = registered_runner(tmp.path()).await;
13030            runner.state.config.graph.timeout_resource_wait = 1;
13031            let winner = runner.state.candidates[0].clone();
13032            let marker = tmp.path().join("fixer-started");
13033            let mut fixer = spec("fixer");
13034            fixer.command = vec![
13035                "sh".into(),
13036                "-c".into(),
13037                format!("echo started >> '{}'; echo done", marker.display()),
13038            ];
13039            runner.roles.fixer = Some(vec![fixer]);
13040            if graft {
13041                let mut loser = winner.clone();
13042                loser.label = 'B';
13043                loser.index = 1;
13044                runner.state.candidates.push(loser);
13045                runner.state.config.graph.graft_runner_up = true;
13046                runner.state.tally = Some(Tally {
13047                    first_choice: BTreeMap::new(),
13048                    borda: BTreeMap::new(),
13049                    winner: winner.label,
13050                    rankings: 1,
13051                    unanimous_initial: true,
13052                    deliberated: false,
13053                    changed_votes: 0,
13054                    unanimous_final: true,
13055                    tie_break: None,
13056                    judges: 1,
13057                    present: 1,
13058                    quorum: 1,
13059                    met_quorum: true,
13060                    uncontested: None,
13061                });
13062                runner.state.judgements.push(crate::run::Judgement {
13063                    judge: 1,
13064                    seat: "judge-1".into(),
13065                    agent: "judge".into(),
13066                    ranking: vec![winner.label, 'B'],
13067                    reasons: BTreeMap::new(),
13068                    confidence: None,
13069                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13070                    order: vec![0, 1],
13071                    failed: None,
13072                    duration_ms: 0,
13073                });
13074            } else {
13075                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13076                    from_run: "earlier".into(),
13077                    generation: 1,
13078                    mode: crate::recovery::Mode::VerifyRepair,
13079                    reasons: Vec::new(),
13080                    reviewed_head: None,
13081                    rounds: Vec::new(),
13082                    open_findings: Vec::new(),
13083                    evidence: None,
13084                    criteria_inherited: true,
13085                    repair: crate::recovery::Repair::NotStarted,
13086                });
13087            }
13088            let reader = crate::repo_lock::SeatLease::for_checkout(
13089                &winner.worktree,
13090                false,
13091                crate::config::test_coordination().as_ref(),
13092            )
13093            .await
13094            .unwrap();
13095            let pause = runner.pause.clone();
13096            let (result, ()) = tokio::join!(
13097                async {
13098                    if graft {
13099                        runner.graft().await
13100                    } else {
13101                        runner.recovery_repair(&winner).await
13102                    }
13103                },
13104                async {
13105                    tokio::time::sleep(Duration::from_millis(100)).await;
13106                    pause.park();
13107                }
13108            );
13109            result.unwrap();
13110            assert!(!marker.exists(), "a blocked seat must not spawn");
13111            // Exercise the persisted state, as a binary replacement/resume does.
13112            runner.state = RunState::load(&runner.state.id).unwrap();
13113            if graft {
13114                assert_eq!(
13115                    runner.state.graft.as_ref().unwrap().outcome,
13116                    GraftOutcome::WaitingForRestore
13117                );
13118            } else {
13119                assert!(matches!(
13120                    runner.state.recovery.as_ref().unwrap().repair,
13121                    crate::recovery::Repair::WaitingForRestore { .. }
13122                ));
13123            }
13124            drop(reader);
13125            runner.pause = Pause::new();
13126            runner.state.resource_block = None;
13127            // Deferred restoration must still respect an external lock owner.
13128            let external =
13129                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13130            assert_eq!(
13131                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13132                0
13133            );
13134            let leftover = winner.worktree.join("leftover");
13135            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13136            let result = if graft {
13137                runner.graft().await
13138            } else {
13139                runner.recovery_repair(&winner).await
13140            };
13141            assert!(
13142                result
13143                    .unwrap_err()
13144                    .is::<crate::repo_lock::ResourceBlocked>()
13145            );
13146            assert!(leftover.exists());
13147            assert!(!marker.exists());
13148            drop(external);
13149            runner.state.resource_block = None;
13150            let result = if graft {
13151                runner.graft().await
13152            } else {
13153                runner.recovery_repair(&winner).await
13154            };
13155            result.unwrap();
13156            assert!(!leftover.exists(), "restore must precede retry");
13157            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13158            if graft {
13159                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13160            } else {
13161                assert!(matches!(
13162                    runner.state.recovery.as_ref().unwrap().repair,
13163                    crate::recovery::Repair::Done { .. }
13164                ));
13165            }
13166        }
13167    }
13168
13169    #[tokio::test]
13170    #[cfg(unix)]
13171    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13172        use std::os::fd::AsRawFd as _;
13173        let tmp = tempfile::tempdir().unwrap();
13174        let mut runner = registered_runner(tmp.path()).await;
13175        runner.state.config.graph.timeout_resource_wait = 1;
13176        let winner = runner.state.candidates[0].clone();
13177        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13178        runner.state.tally = Some(Tally {
13179            first_choice: BTreeMap::new(),
13180            borda: BTreeMap::new(),
13181            winner: winner.label,
13182            rankings: 0,
13183            unanimous_initial: false,
13184            deliberated: false,
13185            changed_votes: 0,
13186            unanimous_final: false,
13187            tie_break: None,
13188            judges: 0,
13189            present: 0,
13190            quorum: 0,
13191            met_quorum: true,
13192            uncontested: Some("fixture".into()),
13193        });
13194        runner.state.graft = Some(GraftRecord {
13195            pre: pre.clone(),
13196            post: None,
13197            lines: None,
13198            limit: None,
13199            agent: None,
13200            outcome: GraftOutcome::Running,
13201            requested: Vec::new(),
13202            adoption: None,
13203            adoption_error: None,
13204        });
13205        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13206        git::commit_all(&winner.worktree, "unfinished graft")
13207            .await
13208            .unwrap();
13209        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13210        let leftover = winner.worktree.join("leftover");
13211        std::fs::write(&leftover, "untrusted edits").unwrap();
13212        let path = runner.state.repo.join(".git/updater.lock");
13213        let external = std::fs::File::create(&path).unwrap();
13214        assert_eq!(
13215            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13216            0
13217        );
13218        assert!(
13219            runner
13220                .graft()
13221                .await
13222                .unwrap_err()
13223                .is::<crate::repo_lock::ResourceBlocked>()
13224        );
13225        assert!(matches!(
13226            runner.state.graft.as_ref().unwrap().outcome,
13227            GraftOutcome::Running
13228        ));
13229        assert_eq!(
13230            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13231            unfinished
13232        );
13233        assert!(leftover.exists());
13234        drop(external);
13235        std::fs::remove_file(&path).unwrap();
13236        std::fs::create_dir(&path).unwrap();
13237        assert!(runner.graft().await.is_err());
13238        assert!(matches!(
13239            runner.state.graft.as_ref().unwrap().outcome,
13240            GraftOutcome::Running
13241        ));
13242        assert_eq!(
13243            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13244            unfinished
13245        );
13246        assert!(leftover.exists());
13247        std::fs::remove_dir(&path).unwrap();
13248        runner.graft().await.unwrap();
13249        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13250        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13251        assert!(!leftover.exists());
13252    }
13253
13254    #[tokio::test]
13255    #[cfg(unix)]
13256    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13257        use std::os::fd::AsRawFd as _;
13258        let tmp = tempfile::tempdir().unwrap();
13259        let mut runner = registered_runner(tmp.path()).await;
13260        runner.state.prep_complete = Some(false);
13261        runner.state.config.graph.timeout_resource_wait = 1;
13262        runner.state.save().unwrap();
13263        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13264        assert_eq!(
13265            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13266            0
13267        );
13268        runner.execute().await.unwrap();
13269        let disk = RunState::load(&runner.state.id).unwrap();
13270        assert!(disk.driver_exited);
13271        assert_eq!(disk.driver_pid, None);
13272        assert_eq!(runner.state.status, RunStatus::Blocked);
13273        assert_eq!(
13274            runner.state.resource_block.as_ref().unwrap().kind,
13275            "repository coordination"
13276        );
13277        assert!(runner.state.prep_failure.is_none());
13278        assert!(runner.state.seats.is_empty());
13279        runner.pause.park();
13280        // Include headroom for Git subprocesses under parallel test load;
13281        // the repository contention deadline itself remains one second.
13282        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13283            .await
13284            .unwrap()
13285            .unwrap();
13286        assert!(runner.state.parked);
13287        assert!(runner.state.prep_failure.is_none());
13288        let disk = RunState::load(&runner.state.id).unwrap();
13289        assert!(disk.driver_exited);
13290        assert_eq!(disk.driver_pid, None);
13291    }
13292
13293    #[tokio::test]
13294    #[cfg(unix)]
13295    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13296        use std::os::fd::AsRawFd as _;
13297        let tmp = tempfile::tempdir().unwrap();
13298        let mut runner = registered_runner(tmp.path()).await;
13299        runner.state.prep_complete = Some(false);
13300        runner.state.config.graph.timeout_resource_wait = 30;
13301        runner.state.save().unwrap();
13302        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13303        assert_eq!(
13304            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13305            0
13306        );
13307        assert!(
13308            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13309                .await
13310                .is_err()
13311        );
13312        let disk = RunState::load(&runner.state.id).unwrap();
13313        assert!(disk.driver_exited);
13314        assert_eq!(disk.driver_pid, None);
13315        assert!(disk.status.resumable());
13316        assert!(runner.state.driver_exited);
13317        assert_eq!(runner.state.driver_pid, None);
13318        runner.state.save().unwrap();
13319        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13320    }
13321
13322    #[tokio::test]
13323    #[cfg(unix)]
13324    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13325        let tmp = tempfile::tempdir().unwrap();
13326        let mut runner = registered_runner(tmp.path()).await;
13327        let winner = runner.state.candidates[0].clone();
13328        let cache = tmp.path().join("cache");
13329        let started = tmp.path().join("pre-gate-started");
13330        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13331        runner.state.config.verify.gate =
13332            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13333        let home = crate::run::home();
13334        let owner =
13335            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13336        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13337            crate::cache::AcquireOutcome::Acquired(g) => g,
13338            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13339        };
13340        let mut gate = Box::pin(runner.run_gate(&winner));
13341        assert!(
13342            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13343                .await
13344                .is_err()
13345        );
13346        assert!(!started.exists(), "pre-gate must wait for the cache too");
13347        // Simulate the wave that already owns the cache entering invoke.
13348        // It must obtain the editor reservation and finish without waiting
13349        // for the gate that is queued behind its cache.
13350        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13351        let editor = crate::repo_lock::SeatLease::checkout_with(
13352            &winner.worktree,
13353            true,
13354            crate::config::test_coordination().as_ref(),
13355            &wait,
13356        )
13357        .await
13358        .unwrap();
13359        drop(editor);
13360        guard.release();
13361        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13362            .await
13363            .unwrap()
13364            .unwrap();
13365        drop(gate);
13366        assert!(outcomes.iter().all(CommandOutcome::ok));
13367        assert!(started.exists());
13368    }
13369
13370    #[tokio::test]
13371    #[cfg(unix)]
13372    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13373        use std::os::fd::AsRawFd as _;
13374
13375        ask_test_home();
13376        let tmp = tempfile::tempdir().unwrap();
13377        let repo = tmp.path().join("repo");
13378        std::fs::create_dir_all(&repo).unwrap();
13379        init_repo(&repo);
13380        let common = repo.join(".git");
13381        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13382        let mut runner = runner_at(RunStatus::Prep);
13383        runner.state.repo = repo.clone();
13384        runner.state.config.coordination = crate::config::test_coordination();
13385        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13386        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13387        runner.state.config.graph.advise = false;
13388        runner.roles.implementers = vec![spec("implement")];
13389        runner.roles.judges = vec![spec("judge")];
13390        runner.prep().await.unwrap();
13391        let winner = runner.state.candidates[0].clone();
13392        // Emulate edits already left by a completed seat. The fixture CLI
13393        // only reports completion; it performs no repository operations.
13394        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13395        let completed = tmp.path().join("seat-completed");
13396        runner.roles.implementers[0].command = vec![
13397            "sh".into(),
13398            "-c".into(),
13399            format!("touch '{}'; echo completed", completed.display()),
13400        ];
13401        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13402        assert_eq!(
13403            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13404            0
13405        );
13406        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13407        let mut implement = Box::pin(runner.implement());
13408        tokio::select! {
13409            result = &mut implement => panic!("rescue must wait: {result:?}"),
13410            result = tokio::time::timeout(Duration::from_secs(10), async {
13411                while !completed.exists() {
13412                    tokio::time::sleep(Duration::from_millis(20)).await;
13413                }
13414            }) => result.unwrap(),
13415        }
13416        assert!(
13417            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13418                .await
13419                .is_err()
13420        );
13421        assert_eq!(
13422            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13423            before
13424        );
13425        assert_eq!(
13426            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13427            0
13428        );
13429        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13430            .await
13431            .unwrap()
13432            .unwrap();
13433        drop(implement);
13434        assert_ne!(
13435            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13436            before
13437        );
13438        assert!(git::is_clean(&winner.worktree).await.unwrap());
13439
13440        let started = tmp.path().join("pre-gate-started");
13441        let proceed = tmp.path().join("pre-gate-proceed");
13442        let checked = tmp.path().join("gate-ran");
13443        runner.state.config.verify.pre_gate = vec![format!(
13444            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13445            started.display(),
13446            proceed.display()
13447        )];
13448        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13449        assert_eq!(
13450            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13451            0
13452        );
13453        let mut gate = Box::pin(runner.run_gate(&winner));
13454        assert!(
13455            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13456                .await
13457                .is_err()
13458        );
13459        assert!(!started.exists());
13460        assert!(!checked.exists());
13461        assert!(git::is_clean(&winner.worktree).await.unwrap());
13462        assert_eq!(
13463            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13464            0
13465        );
13466        tokio::select! {
13467            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13468            result = tokio::time::timeout(Duration::from_secs(10), async {
13469                while !started.exists() {
13470                    tokio::time::sleep(Duration::from_millis(20)).await;
13471                }
13472            }) => result.unwrap(),
13473        }
13474        assert_eq!(
13475            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13476            -1,
13477            "mechanical edits must hold the real lock"
13478        );
13479        std::fs::write(proceed, "ready").unwrap();
13480        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13481            .await
13482            .unwrap()
13483            .unwrap();
13484        drop(gate);
13485        assert!(outcomes.iter().all(CommandOutcome::ok));
13486        assert!(checked.exists());
13487        assert!(runner.state.pre_gate_commit.is_some());
13488        assert!(git::is_clean(&winner.worktree).await.unwrap());
13489        assert_eq!(
13490            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13491            0,
13492            "gate completion must release coordination"
13493        );
13494        drop(outsider);
13495        std::fs::remove_file(common.join("updater.lock")).unwrap();
13496        std::fs::create_dir(common.join("updater.lock")).unwrap();
13497        std::fs::remove_file(&started).unwrap();
13498        std::fs::remove_file(&checked).unwrap();
13499        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13500        let error = runner.run_gate(&winner).await.unwrap_err();
13501        assert!(
13502            error
13503                .to_string()
13504                .contains("acquire mandatory repository lock")
13505        );
13506        assert!(
13507            !started.exists(),
13508            "failed acquisition must prevent pre-gate"
13509        );
13510        assert!(!checked.exists(), "failed acquisition must prevent gate");
13511        assert!(git::is_clean(&winner.worktree).await.unwrap());
13512        assert_eq!(
13513            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13514            head
13515        );
13516    }
13517
13518    #[tokio::test]
13519    #[cfg(unix)]
13520    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13521        use std::os::fd::AsRawFd as _;
13522        use std::time::Duration;
13523
13524        ask_test_home();
13525        let tmp = tempfile::tempdir().unwrap();
13526        let repo = tmp.path().join("repo");
13527        std::fs::create_dir_all(&repo).unwrap();
13528        init_repo(&repo);
13529        let common = repo.join(".git");
13530        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13531        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13532        assert_eq!(
13533            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13534            0
13535        );
13536        let mut runner = runner_at(RunStatus::Prep);
13537        runner.state.repo = repo.clone();
13538        runner.state.config.coordination = crate::config::test_coordination();
13539        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13540        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13541        runner.state.config.graph.advise = true;
13542        runner.state.config.graph.advisors = 1;
13543        runner.roles.implementers = vec![spec("implement")];
13544        runner.roles.judges = vec![spec("judge")];
13545        let root = runner.state.worktree_root();
13546        let started = tmp.path().join("setup-started");
13547        let proceed = tmp.path().join("setup-proceed");
13548        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13549            run: Some(format!(
13550                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13551                started.display(),
13552                proceed.display()
13553            )),
13554            ..Default::default()
13555        }];
13556        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13557        let config = git::git(&repo, &["config", "--local", "--list"])
13558            .await
13559            .unwrap();
13560        let mut prep = Box::pin(runner.prep());
13561        assert!(
13562            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13563                .await
13564                .is_err()
13565        );
13566        assert!(
13567            !root.exists(),
13568            "contention must prevent even checkout creation"
13569        );
13570        assert!(!started.exists(), "contention must prevent setup");
13571        assert_eq!(
13572            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13573            branches
13574        );
13575        assert_eq!(
13576            git::git(&repo, &["config", "--local", "--list"])
13577                .await
13578                .unwrap(),
13579            config
13580        );
13581        drop(lock);
13582        tokio::select! {
13583            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13584            result = tokio::time::timeout(Duration::from_secs(10), async {
13585                while !started.exists() {
13586                    tokio::time::sleep(Duration::from_millis(20)).await;
13587                }
13588            }) => result.unwrap(),
13589        }
13590        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13591        assert_eq!(
13592            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13593            -1
13594        );
13595        assert!(
13596            tokio::time::timeout(
13597                Duration::from_millis(100),
13598                crate::repo_lock::SeatLease::for_checkout(
13599                    &repo,
13600                    true,
13601                    crate::config::test_coordination().as_ref()
13602                )
13603            )
13604            .await
13605            .is_err(),
13606            "preparation must also exclude local editors"
13607        );
13608        std::fs::write(proceed, "ready").unwrap();
13609        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13610            .await
13611            .unwrap()
13612            .unwrap();
13613        drop(prep);
13614        assert_eq!(runner.state.prep_complete, Some(true));
13615        assert!(root.join("judge-1").exists());
13616        assert!(root.join("advisor-1").exists());
13617        assert_eq!(
13618            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13619            0
13620        );
13621    }
13622
13623    #[tokio::test]
13624    #[cfg(unix)]
13625    async fn preparation_lock_failure_prevents_repository_changes() {
13626        ask_test_home();
13627        let tmp = tempfile::tempdir().unwrap();
13628        let repo = tmp.path().join("repo");
13629        std::fs::create_dir_all(&repo).unwrap();
13630        init_repo(&repo);
13631        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13632        // A directory at the required lock path makes opening the real lock
13633        // fail, rather than merely exercising an invalid registration.
13634        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13635        let mut runner = runner_at(RunStatus::Prep);
13636        runner.state.repo = repo.clone();
13637        runner.state.config.coordination = crate::config::test_coordination();
13638        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13639        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13640        runner.roles.implementers = vec![spec("implement")];
13641        let error = runner.prep().await.unwrap_err();
13642        assert!(
13643            error
13644                .to_string()
13645                .contains("acquire mandatory repository lock")
13646        );
13647        assert!(!runner.state.worktree_root().exists());
13648        assert!(runner.state.candidates.is_empty());
13649        assert!(!repo.join(".git/config.worktree").exists());
13650        assert_eq!(
13651            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13652            "* main"
13653        );
13654    }
13655
13656    #[tokio::test]
13657    async fn preparation_records_creation_intent_before_git_can_fail() {
13658        ask_test_home();
13659        let tmp = tempfile::tempdir().unwrap();
13660        let repo = tmp.path().join("repo");
13661        std::fs::create_dir_all(&repo).unwrap();
13662        init_repo(&repo);
13663        let mut runner = runner_at(RunStatus::Prep);
13664        runner.state.repo = repo.clone();
13665        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13666        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13667        runner.state.config.blind.commit_msg_hook = false;
13668        runner.roles.implementers = vec![spec("one")];
13669        let label = blind::assign_labels(1, runner.state.seed)[0];
13670        let branch = runner.state.branch_for(label);
13671        // A conflicting branch makes Git refuse creation before a candidate
13672        // can be registered. The durable intent must still identify it.
13673        git::git(&repo, &["branch", &branch]).await.unwrap();
13674        assert!(runner.prep().await.is_err());
13675        let saved = RunState::load(&runner.state.id).unwrap();
13676        assert!(saved.candidates.is_empty());
13677        let event = saved
13678            .events
13679            .iter()
13680            .find(|e| e.node == "prep-intent")
13681            .unwrap();
13682        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13683        assert_eq!(intent["branch"], branch);
13684        assert_eq!(intent["start"], runner.state.base_commit);
13685        assert_eq!(
13686            intent["worktree"],
13687            runner
13688                .state
13689                .worktree_root()
13690                .join(format!("cand-{label}"))
13691                .to_string_lossy()
13692                .as_ref()
13693        );
13694        assert_eq!(intent["picks"], serde_json::json!([]));
13695    }
13696
13697    #[tokio::test]
13698    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13699        ask_test_home();
13700        let tmp = tempfile::tempdir().unwrap();
13701        let repo = tmp.path().join("repo");
13702        std::fs::create_dir_all(&repo).unwrap();
13703        init_repo(&repo);
13704        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13705        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13706        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13707        git::git(&repo, &["add", "."]).await.unwrap();
13708        git::git(&repo, &["commit", "-m", "required seed"])
13709            .await
13710            .unwrap();
13711        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13712        git::git(&repo, &["checkout", "main"]).await.unwrap();
13713        let mut runner = runner_at(RunStatus::Prep);
13714        runner.state.repo = repo.clone();
13715        runner.state.base_commit = base.clone();
13716        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13717        runner.state.config.graph.advise = false;
13718        runner.state.config.blind.commit_msg_hook = false;
13719        runner.state.seeds = vec![refs::Seed {
13720            token: seed.clone(),
13721            sha: seed.clone(),
13722            kind: refs::SeedKind::Unmerged,
13723            branch: false,
13724            detail: String::new(),
13725        }];
13726        runner.roles.implementers = vec![spec("one"), spec("two")];
13727        runner.roles.judges = vec![spec("judge")];
13728        let label = blind::assign_labels(2, runner.state.seed)[0];
13729        let branch = runner.state.branch_for(label);
13730        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13731        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13732            .await
13733            .unwrap();
13734        runner.state.candidates.push(Candidate {
13735            index: 0,
13736            label,
13737            agent: "one".into(),
13738            branch,
13739            worktree,
13740            summary: String::new(),
13741            stat: String::new(),
13742            files: 0,
13743            commits: 0,
13744            empty: false,
13745            failed: None,
13746            verified_noop: None,
13747            duration_ms: 0,
13748            folded: false,
13749        });
13750        runner.state.save().unwrap();
13751        // Reload the checkpoint immediately before its first cherry-pick.
13752        runner.state = RunState::load(&runner.state.id).unwrap();
13753        assert_eq!(runner.state.prep_complete, Some(false));
13754        runner.prep().await.unwrap();
13755        assert_eq!(runner.state.prep_complete, Some(true));
13756        assert_eq!(runner.state.candidates.len(), 2);
13757        for candidate in &runner.state.candidates {
13758            assert_eq!(
13759                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13760                git::tree_of(&repo, &seed).await.unwrap()
13761            );
13762            assert_eq!(
13763                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13764                "required commit"
13765            );
13766        }
13767        assert!(runner.state.worktree_root().join("judge-1").exists());
13768        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13769        runner.prep().await.unwrap();
13770        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13771
13772        runner.state.prep_complete = Some(false);
13773        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13774            run: Some(format!(
13775                "test -f '{}'",
13776                tmp.path().join("dependency-ready").display()
13777            )),
13778            ..Default::default()
13779        }];
13780        assert!(runner.prep().await.is_err());
13781        assert!(runner.state.prep_failure.is_some());
13782        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13783        runner.state = RunState::load(&runner.state.id).unwrap();
13784        assert!(
13785            runner.prep().await.is_err(),
13786            "correcting the cause alone must not auto-retry"
13787        );
13788        let failed_at = runner.state.events.last().unwrap().at;
13789        assert!(
13790            runner.retry_preparation(failed_at).is_err(),
13791            "old approvals cannot unlock the hold"
13792        );
13793        let mut task = crate::queue::Task::new(
13794            "retry".into(),
13795            "retry setup".into(),
13796            repo.clone(),
13797            crate::queue::Source::Human,
13798        );
13799        task.start(runner.state.id.clone());
13800        task.hold_machine(Some("setup failed".into()));
13801        task.release_operator("cli", None);
13802        runner
13803            .retry_preparation(task.resumption.as_ref().unwrap().at)
13804            .unwrap();
13805        runner.prep().await.unwrap();
13806        assert!(runner.state.prep_failure.is_none());
13807        assert_eq!(runner.state.prep_complete, Some(true));
13808        assert_eq!(runner.state.candidates.len(), 2);
13809        assert!(runner.state.worktree_root().join("judge-1").exists());
13810        for candidate in &runner.state.candidates {
13811            assert_eq!(
13812                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13813                git::tree_of(&repo, &seed).await.unwrap()
13814            );
13815        }
13816    }
13817
13818    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13819        let mut heads = Vec::new();
13820        for candidate in candidates {
13821            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13822        }
13823        heads
13824    }
13825
13826    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13827        ask_test_home();
13828        let tmp = tempfile::tempdir().unwrap();
13829        let repo = tmp.path().join("repo");
13830        std::fs::create_dir(&repo).unwrap();
13831        init_repo(&repo);
13832        let mut runner = runner_at(RunStatus::Reviewing);
13833        runner.state.repo = repo.clone();
13834        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13835        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13836        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13837            let branch = format!("candidate-{label}");
13838            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13839            git::worktree_add_branch(&repo, &worktree, &branch, "main")
13840                .await
13841                .unwrap();
13842            if label == 'B' {
13843                std::fs::write(worktree.join("change"), "winner").unwrap();
13844                git::rescue_commit(&worktree, "candidate change")
13845                    .await
13846                    .unwrap();
13847            }
13848            runner.state.candidates.push(Candidate {
13849                index,
13850                label,
13851                agent: "alpha".into(),
13852                branch,
13853                worktree,
13854                summary: "completed reply".into(),
13855                stat: String::new(),
13856                files: usize::from(label == 'B'),
13857                commits: usize::from(label == 'B'),
13858                empty: label == 'A',
13859                failed: None,
13860                verified_noop: None,
13861                duration_ms: 1,
13862                folded: false,
13863            });
13864        }
13865        runner.state.judge_skipped = true;
13866        runner.state.tally = Some(Tally {
13867            first_choice: BTreeMap::new(),
13868            borda: BTreeMap::new(),
13869            winner: 'B',
13870            rankings: 0,
13871            unanimous_initial: false,
13872            deliberated: false,
13873            changed_votes: 0,
13874            unanimous_final: false,
13875            tie_break: None,
13876            judges: 0,
13877            present: 0,
13878            quorum: 0,
13879            met_quorum: true,
13880            uncontested: Some("only B produced a change".into()),
13881        });
13882        runner.state.save().unwrap();
13883        (tmp, runner)
13884    }
13885
13886    #[tokio::test]
13887    #[cfg(unix)]
13888    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13889        for (solo, failed_checkpoint) in
13890            [(true, false), (false, false), (true, true), (false, true)]
13891        {
13892            let (_tmp, mut runner) = preservation_resume_fixture().await;
13893            if solo {
13894                runner.state.candidates.retain(|c| c.label == 'A');
13895            }
13896            runner.state.candidates[0].empty = false;
13897            runner.state.candidates[0].summary.clear();
13898            runner.state.tally = None;
13899            runner.state.judge_skipped = false;
13900            runner.state.status = RunStatus::Implementing;
13901            runner.state.config.disk.min_free_bytes = u64::MAX;
13902            let mut implementer = spec("alpha");
13903            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13904            runner.roles.implementers = vec![implementer];
13905            let patch = runner.state.dir().join("artifacts/cand-A.patch");
13906            if failed_checkpoint {
13907                std::fs::create_dir_all(&patch).unwrap();
13908            }
13909            runner.state.save().unwrap();
13910            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13911            runner.state = RunState::load(&runner.state.id).unwrap();
13912            assert_eq!(runner.state.status, RunStatus::Blocked);
13913            assert!(runner.state.interrupted_implementations.contains(&'A'));
13914            assert_eq!(runner.state.candidates[0].commits, 0);
13915            assert!(!runner.state.candidates[0].empty);
13916            assert!(runner.state.candidates[0].failed.is_none());
13917            if failed_checkpoint {
13918                assert!(runner.state.candidate_preservation.contains_key(&'A'));
13919                std::fs::remove_dir(patch).unwrap();
13920            }
13921            runner.state.resource_block = None;
13922            runner.state.config.disk.min_free_bytes = 0;
13923            runner.roles.implementers[0].command = vec![
13924                "sh".into(),
13925                "-c".into(),
13926                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13927            ];
13928            runner.implement().await.unwrap();
13929            assert!(runner.state.interrupted_implementations.is_empty());
13930            assert!(runner.state.candidate_preservation.is_empty());
13931            assert_eq!(runner.state.candidates[0].commits, 1);
13932            assert!(!runner.state.candidates[0].empty);
13933            assert!(
13934                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13935                    .unwrap()
13936                    .contains("retried empty interrupted seat")
13937            );
13938            if !solo {
13939                assert_eq!(runner.state.candidates[1].commits, 1);
13940            }
13941        }
13942    }
13943
13944    #[tokio::test]
13945    #[cfg(unix)]
13946    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13947        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13948            let (_tmp, mut runner) = preservation_resume_fixture().await;
13949            runner.state.candidates.retain(|c| c.label == 'A');
13950            runner.state.candidates[0].empty = false;
13951            runner.state.candidates[0].summary.clear();
13952            runner.state.tally = None;
13953            runner.state.judge_skipped = false;
13954            runner.state.status = RunStatus::Implementing;
13955            // Force the real post-spawn watchdog, independently of available
13956            // disk space. Dirty/committed work predates its first measurement.
13957            runner.state.config.disk.min_free_bytes = u64::MAX;
13958            let wt = runner.state.candidates[0].worktree.clone();
13959            std::fs::write(
13960                wt.join("interrupted-source"),
13961                "retain implementation work\n",
13962            )
13963            .unwrap();
13964            if committed {
13965                git::rescue_commit(&wt, "implementation checkpoint")
13966                    .await
13967                    .unwrap();
13968            }
13969            let lock = git::git(
13970                &wt,
13971                &[
13972                    "rev-parse",
13973                    "--path-format=absolute",
13974                    "--git-path",
13975                    "index.lock",
13976                ],
13977            )
13978            .await
13979            .unwrap();
13980            if block_rescue {
13981                std::fs::write(&lock, "fixture-owned lock").unwrap();
13982            }
13983            let mut implementer = spec("alpha");
13984            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13985            runner.roles.implementers = vec![implementer];
13986            runner.state.save().unwrap();
13987            let result = runner.implement().await;
13988            assert_eq!(result.is_err(), block_rescue);
13989            assert_eq!(
13990                runner.state.resource_block.as_ref().unwrap().kind,
13991                "disk pressure"
13992            );
13993            assert!(
13994                runner
13995                    .state
13996                    .events
13997                    .iter()
13998                    .any(|e| e.message.contains("agent execution interrupted"))
13999            );
14000            assert!(
14001                runner
14002                    .state
14003                    .dir()
14004                    .join("artifacts/candidate-A-recovery/staged.patch")
14005                    .exists()
14006            );
14007            if block_rescue {
14008                let saved = RunState::load(&runner.state.id).unwrap();
14009                assert!(saved.candidate_preservation.contains_key(&'A'));
14010                assert!(!saved.candidates[0].empty);
14011                assert_eq!(
14012                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14013                    "retain implementation work\n"
14014                );
14015                std::fs::remove_file(lock).unwrap();
14016                runner.state = saved;
14017            } else {
14018                assert_eq!(runner.state.status, RunStatus::Blocked);
14019                assert_eq!(runner.state.candidates[0].commits, 1);
14020            }
14021            // Recovery must save the candidate before deciding whether another
14022            // implementation is needed. No implementer is available on retry.
14023            runner.roles.implementers.clear();
14024            runner.state.config.disk.min_free_bytes = 0;
14025            runner.state.resource_block = None;
14026            runner.implement().await.unwrap();
14027            assert!(runner.state.candidate_preservation.is_empty());
14028            assert_eq!(runner.state.candidates[0].commits, 1);
14029            assert!(!runner.state.candidates[0].empty);
14030            assert!(git::is_clean(&wt).await.unwrap());
14031            assert!(
14032                crate::run::read_artifact(&runner.state, "cand-A.patch")
14033                    .unwrap()
14034                    .contains("retain implementation work")
14035            );
14036        }
14037    }
14038
14039    #[tokio::test]
14040    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14041        let (_tmp, mut runner) = preservation_resume_fixture().await;
14042        // Schema 15 already confirmed the unfolded no-change result.
14043        runner.implement().await.unwrap();
14044        assert!(runner.state.candidate_preservation.is_empty());
14045        assert!(
14046            !runner
14047                .state
14048                .dir()
14049                .join("artifacts/candidate-A-recovery")
14050                .exists()
14051        );
14052        runner.fold_losers().await.unwrap();
14053        assert!(!runner.state.candidates[0].worktree.exists());
14054        for schema in [14, crate::run::SCHEMA] {
14055            runner.state.schema = schema;
14056            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14057            runner.state.save().unwrap();
14058            runner.state = RunState::load(&runner.state.id).unwrap();
14059            runner.implement().await.unwrap();
14060            assert!(runner.state.candidate_preservation.is_empty());
14061            assert!(runner.state.candidates[0].folded);
14062            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14063        }
14064    }
14065
14066    #[tokio::test]
14067    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14068        let (_tmp, mut runner) = preservation_resume_fixture().await;
14069        let wt = runner.state.candidates[0].worktree.clone();
14070        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14071        runner.state.schema = 14;
14072        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14073        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14074        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14075        for _ in 0..2 {
14076            assert!(runner.implement().await.is_err());
14077            assert_eq!(runner.state.status, RunStatus::Blocked);
14078            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14079            assert!(
14080                runner.state.candidates[0].empty,
14081                "old verdict must remain unusable"
14082            );
14083            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14084            assert!(runner.state.judge_skipped);
14085            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14086            assert!(
14087                crate::run::read_artifact(&runner.state, "cand-A.patch")
14088                    .unwrap()
14089                    .contains("owned edits")
14090            );
14091            assert!(wt.exists());
14092            assert_eq!(
14093                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14094                    .await
14095                    .unwrap(),
14096                1
14097            );
14098            runner.state = RunState::load(&runner.state.id).unwrap();
14099        }
14100    }
14101
14102    #[tokio::test]
14103    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14104        let (_tmp, mut runner) = preservation_resume_fixture().await;
14105        let wt = runner.state.candidates[0].worktree.clone();
14106        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14107        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14108        let index_lock = PathBuf::from(
14109            git::git(
14110                &wt,
14111                &[
14112                    "rev-parse",
14113                    "--path-format=absolute",
14114                    "--git-path",
14115                    "index.lock",
14116                ],
14117            )
14118            .await
14119            .unwrap(),
14120        );
14121        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14122        runner.state.schema = 14;
14123        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14124        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14125        assert!(runner.implement().await.is_err());
14126        assert!(runner.after_implement().is_err());
14127        assert!(runner.fold_losers().await.is_err());
14128        assert_eq!(
14129            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14130            "failed seat edits"
14131        );
14132        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14133        runner.state = RunState::load(&runner.state.id).unwrap();
14134        std::fs::remove_file(index_lock).unwrap();
14135        // Recovery finds changes that the old uncontested result never judged.
14136        assert!(runner.implement().await.is_err());
14137        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14138        assert!(runner.state.candidates[0].empty);
14139        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14140        assert!(
14141            crate::run::read_artifact(&runner.state, "cand-A.patch")
14142                .unwrap()
14143                .contains("failed seat edits")
14144        );
14145        assert!(wt.exists());
14146    }
14147
14148    #[tokio::test]
14149    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14150        let (_tmp, mut runner) = preservation_resume_fixture().await;
14151        let repo = runner.state.repo.clone();
14152        let common = git::git(
14153            &repo,
14154            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14155        )
14156        .await
14157        .unwrap();
14158        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14159        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14160        assert!(runner.fold_losers().await.is_err());
14161        assert!(!runner.state.candidates[0].worktree.exists());
14162        assert!(!runner.state.candidates[0].folded);
14163        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14164        runner.state = RunState::load(&runner.state.id).unwrap();
14165        std::fs::remove_file(branch_lock).unwrap();
14166        let judge = runner.state.worktree_root().join("judge-1");
14167        git::worktree_add_detached(&repo, &judge, "main")
14168            .await
14169            .unwrap();
14170        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14171        runner.roles.judges = vec![spec("judge")];
14172        assert!(runner.fold_losers().await.is_err());
14173        runner.state = RunState::load(&runner.state.id).unwrap();
14174        assert!(
14175            runner.state.candidates[0].folded,
14176            "candidate progress saved before judge failure"
14177        );
14178        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14179        assert!(judge.join("uncommitted").exists());
14180        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14181        runner.fold_losers().await.unwrap();
14182        assert!(!judge.exists());
14183        runner.fold_losers().await.unwrap();
14184    }
14185
14186    #[tokio::test]
14187    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14188        let (_tmp, mut runner) = preservation_resume_fixture().await;
14189        runner.state.candidates.retain(|c| c.label == 'B');
14190        runner.state.tally = None;
14191        runner.state.judge_skipped = false;
14192        runner.state.prep_complete = Some(true);
14193        runner.state.config.graph.advise = false;
14194        runner.state.config.graph.selected_implementer = Some("alpha".into());
14195        let wt = runner.state.candidates[0].worktree.clone();
14196        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14197        runner
14198            .state
14199            .candidate_preservation
14200            .insert('B', "awaiting persistence".into());
14201        let index_lock = git::git(
14202            &wt,
14203            &[
14204                "rev-parse",
14205                "--path-format=absolute",
14206                "--git-path",
14207                "index.lock",
14208            ],
14209        )
14210        .await
14211        .unwrap();
14212        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14213        runner.state.save().unwrap();
14214        assert!(runner.execute_graph().await.is_err());
14215        assert_eq!(runner.state.status, RunStatus::Blocked);
14216        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14217        std::fs::remove_file(index_lock).unwrap();
14218        // Park at the next graph boundary: this tests the real selected-seat
14219        // hold guard without dispatching reviewers or publishing anything.
14220        for _ in 0..2 {
14221            runner.state = RunState::load(&runner.state.id).unwrap();
14222            runner.pause.park();
14223            runner.execute_graph().await.unwrap();
14224            assert_eq!(runner.state.status, RunStatus::Implementing);
14225            assert!(runner.state.candidate_preservation.is_empty());
14226            assert_eq!(runner.state.candidates[0].commits, 2);
14227            assert!(runner.state.tally.is_none());
14228        }
14229        runner.implement().await.unwrap();
14230        assert_eq!(runner.state.status, RunStatus::Judging);
14231        // A genuine selected-implementer hold must still require release.
14232        runner.state.status = RunStatus::Blocked;
14233        runner.state.candidates[0].failed = Some("implementation failed".into());
14234        runner.state.save().unwrap();
14235        let error = runner.execute_graph().await.unwrap_err();
14236        assert!(
14237            error
14238                .to_string()
14239                .contains("selected implementation is held")
14240        );
14241    }
14242
14243    #[tokio::test]
14244    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14245        let (_tmp, mut runner) = preservation_resume_fixture().await;
14246        let repo = runner.state.repo.clone();
14247        let candidate = runner.state.candidates[0].clone();
14248        // Remove only the disposable fixture directory, leaving Git's linked
14249        // worktree registration and branch intact, as in a vanished checkout.
14250        let registered = git::worktree_holding(&repo, &candidate.branch)
14251            .await
14252            .unwrap()
14253            .unwrap();
14254        assert_eq!(
14255            registered.canonicalize().unwrap(),
14256            candidate.worktree.canonicalize().unwrap()
14257        );
14258        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14259        assert_eq!(
14260            git::worktree_holding(&repo, &candidate.branch)
14261                .await
14262                .unwrap(),
14263            Some(registered)
14264        );
14265        assert!(
14266            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14267            "the stale registration initially keeps the branch checked out"
14268        );
14269        runner.fold_losers().await.unwrap();
14270        assert!(runner.state.candidates[0].folded);
14271        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14272        assert_eq!(
14273            git::worktree_holding(&repo, &candidate.branch)
14274                .await
14275                .unwrap(),
14276            None
14277        );
14278        runner.state = RunState::load(&runner.state.id).unwrap();
14279        runner.fold_losers().await.unwrap();
14280    }
14281
14282    #[tokio::test]
14283    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14284        let (_tmp, mut runner) = preservation_resume_fixture().await;
14285        let repo = runner.state.repo.clone();
14286        let candidate = runner.state.candidates[0].clone();
14287        let winner = runner.state.candidates[1].clone();
14288        let registered = git::worktree_holding(&repo, &candidate.branch)
14289            .await
14290            .unwrap()
14291            .unwrap();
14292        assert_eq!(
14293            registered.canonicalize().unwrap(),
14294            candidate.worktree.canonicalize().unwrap()
14295        );
14296        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14297        assert_eq!(
14298            git::worktree_holding(&repo, &candidate.branch)
14299                .await
14300                .unwrap(),
14301            Some(registered)
14302        );
14303        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14304
14305        for _ in 0..2 {
14306            fold_run(&mut runner.state, false, &crate::run::home())
14307                .await
14308                .unwrap();
14309            runner.state = RunState::load(&runner.state.id).unwrap();
14310            assert!(runner.state.candidates[0].folded);
14311            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14312            assert_eq!(
14313                git::worktree_holding(&repo, &candidate.branch)
14314                    .await
14315                    .unwrap(),
14316                None
14317            );
14318            assert!(!runner.state.candidates[1].folded);
14319            assert!(winner.worktree.exists());
14320            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14321        }
14322    }
14323
14324    #[tokio::test]
14325    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14326        ask_test_home();
14327        let tmp = tempfile::tempdir().unwrap();
14328        let repo = tmp.path().join("repo");
14329        std::fs::create_dir(&repo).unwrap();
14330        init_repo(&repo);
14331        let mut runner = runner_at(RunStatus::Implementing);
14332        runner.state.repo = repo.clone();
14333        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14334        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14335        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14336            let branch = format!("candidate-{label}");
14337            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14338            git::worktree_add_branch(&repo, &wt, &branch, "main")
14339                .await
14340                .unwrap();
14341            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14342            git::git(&wt, &["add", "README.md"]).await.unwrap();
14343            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14344            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14345            runner.state.candidates.push(Candidate {
14346                index,
14347                label,
14348                agent: "alpha".to_owned(),
14349                branch,
14350                worktree: wt,
14351                summary: "completed implementation".to_owned(),
14352                stat: String::new(),
14353                files: 0,
14354                commits: 0,
14355                empty: false,
14356                failed: None,
14357                verified_noop: None,
14358                duration_ms: 42,
14359                folded: false,
14360            });
14361            runner
14362                .state
14363                .candidate_preservation
14364                .insert(label, "awaiting persistence".to_owned());
14365        }
14366        let wt = runner.state.candidates[0].worktree.clone();
14367        let index_lock = git::git(
14368            &wt,
14369            &[
14370                "rev-parse",
14371                "--path-format=absolute",
14372                "--git-path",
14373                "index.lock",
14374            ],
14375        )
14376        .await
14377        .unwrap();
14378        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14379        runner.state.save().unwrap();
14380        assert!(runner.persist_candidates().await.is_err());
14381        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14382        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14383        assert_eq!(
14384            runner.state.candidates[1].commits, 1,
14385            "other results are still collected"
14386        );
14387        assert!(
14388            !runner.state.candidates[0].empty,
14389            "failed rescue is not no change"
14390        );
14391        assert_eq!(
14392            runner.state.candidates[0].summary,
14393            "completed implementation"
14394        );
14395        assert!(runner.after_implement().is_err());
14396        let home = tmp.path().join("fold-home");
14397        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14398        assert!(wt.join("new-file").exists());
14399        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14400        assert!(recovery.join("staged.patch").exists());
14401        assert!(recovery.join("unstaged.patch").exists());
14402        assert_eq!(
14403            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14404            "untracked edits\n"
14405        );
14406        runner.state.begin_pass();
14407        runner.state.end_pass_parked(ParkCause::Upgrade {
14408            from: "1.0".into(),
14409            to: Some("1.1".into()),
14410        });
14411        runner.state.save().unwrap();
14412        runner.state = RunState::load(&runner.state.id).unwrap();
14413        assert!(matches!(
14414            runner.state.passes.last().unwrap().end,
14415            Some(crate::run::PassEnd::Parked {
14416                cause: ParkCause::Upgrade { .. }
14417            })
14418        ));
14419        std::fs::remove_file(index_lock).unwrap();
14420        // No implementer is configured: a persistence retry cannot dispatch a seat.
14421        runner.implement().await.unwrap();
14422        assert!(runner.state.candidate_preservation.is_empty());
14423        assert!(
14424            runner
14425                .state
14426                .candidates
14427                .iter()
14428                .all(|c| c.commits == 1 && !c.empty)
14429        );
14430        assert!(git::is_clean(&wt).await.unwrap());
14431        assert!(
14432            crate::run::read_artifact(&runner.state, "cand-A.patch")
14433                .unwrap()
14434                .contains("later tracked edits")
14435        );
14436    }
14437
14438    #[tokio::test]
14439    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14440        ask_test_home();
14441        let tmp = tempfile::tempdir().unwrap();
14442        let repo = tmp.path().join("repo");
14443        std::fs::create_dir(&repo).unwrap();
14444        init_repo(&repo);
14445        let wt = tmp.path().join("candidate");
14446        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14447            .await
14448            .unwrap();
14449        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14450        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14451        assert!(wt.join("uncommitted").exists());
14452        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14453        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14454        let mut runner = runner_at(RunStatus::Implementing);
14455        runner.state.repo = repo.clone();
14456        runner.state.base_commit = "missing-base".to_owned();
14457        runner.state.candidates.push(Candidate {
14458            index: 0,
14459            label: 'A',
14460            agent: "alpha".into(),
14461            branch: "candidate".into(),
14462            worktree: wt.clone(),
14463            summary: "completed".into(),
14464            stat: String::new(),
14465            files: 0,
14466            commits: 0,
14467            empty: false,
14468            failed: None,
14469            verified_noop: None,
14470            duration_ms: 1,
14471            folded: false,
14472        });
14473        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14474        assert!(
14475            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14476                .await
14477                .is_err()
14478        );
14479        assert!(!runner.state.candidates[0].folded);
14480        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14481        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14482        runner
14483            .state
14484            .candidate_preservation
14485            .insert('A', "pending".into());
14486        assert!(runner.persist_candidates().await.is_err());
14487        assert!(!runner.state.candidates[0].empty);
14488        assert!(!runner.state.candidates[0].folded);
14489        assert!(wt.exists());
14490        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14491        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14492        std::fs::create_dir_all(&patch_path).unwrap();
14493        assert!(
14494            runner.persist_candidates().await.is_err(),
14495            "artifact failure stays pending"
14496        );
14497        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14498        assert!(!runner.state.candidates[0].empty);
14499        std::fs::remove_dir(patch_path).unwrap();
14500        runner.persist_candidates().await.unwrap();
14501        assert!(
14502            runner.state.candidates[0].empty,
14503            "only successful inspection confirms no change"
14504        );
14505    }
14506
14507    fn init_repo(dir: &Path) {
14508        let run = |args: &[&str]| {
14509            let out = std::process::Command::new("git")
14510                .args(args)
14511                .current_dir(dir)
14512                .quiet()
14513                .output()
14514                .expect("spawn git");
14515            assert!(
14516                out.status.success(),
14517                "git {args:?} failed: {}",
14518                String::from_utf8_lossy(&out.stderr)
14519            );
14520        };
14521        run(&["init", "-b", "main"]);
14522        run(&["config", "user.name", "magi test"]);
14523        run(&["config", "user.email", "magi@example.com"]);
14524        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14525        run(&["add", "-A"]);
14526        run(&["commit", "-m", "init"]);
14527    }
14528
14529    // `settle_questions` is what closes the ghost the phone showed: a run's
14530    // seat asked something, the run then ended, and nothing was left to
14531    // abandon the question it left `open`. `HOME` is a process-wide
14532    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14533    // first time it runs in the binary — every test below still reaches the
14534    // same directory whichever call won, and each gets its own run id from
14535    // `RunState::new`, so they never collide there.
14536    fn ask_test_home() {
14537        crate::run::pin_test_home();
14538    }
14539
14540    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14541    /// reads nothing else off it.
14542    fn runner_at(status: RunStatus) -> Runner {
14543        let mut state = RunState::new(
14544            PathBuf::from("/nonexistent/repo"),
14545            "main".to_owned(),
14546            "deadbeef".to_owned(),
14547            "task".to_owned(),
14548            Config::default(),
14549        );
14550        // These fixtures share one home and save concurrently. Short clock
14551        // IDs can collide, making an ownership test read another test's run.
14552        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14553        state.id = format!(
14554            "20000101-000000-{:04x}",
14555            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14556        );
14557        state.status = status;
14558        Runner {
14559            state,
14560            roles: ResolvedRoles {
14561                implementers: Vec::new(),
14562                judges: Vec::new(),
14563                reviewers: Vec::new(),
14564                fixer: None,
14565                conductor: conductor(),
14566                implementer_roster: Vec::new(),
14567                judge_roster: Vec::new(),
14568                reviewer_roster: Vec::new(),
14569            },
14570            sem: Arc::new(Semaphore::new(1)),
14571            pause: Pause::new(),
14572            interrupt: Pause::new(),
14573        }
14574    }
14575
14576    #[test]
14577    fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14578        crate::run::pin_test_home();
14579        let mut runner = runner_at(RunStatus::Gating);
14580        runner.state.save().unwrap();
14581        // run.json can commit, but the subsequent lease metadata write fails.
14582        // Cleanup must run after releasing the short ownership lock.
14583        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14584        assert!(runner.claim_driver().is_err());
14585        assert!(runner.state.driver_exited);
14586        assert_eq!(runner.state.driver_pid, None);
14587        let disk = RunState::load(&runner.state.id).unwrap();
14588        assert!(disk.driver_exited);
14589        assert_eq!(disk.driver_pid, None);
14590        assert_eq!(disk.status, RunStatus::Gating);
14591    }
14592
14593    #[tokio::test]
14594    async fn an_early_graph_error_releases_its_driver() {
14595        crate::run::pin_test_home();
14596        let mut runner = runner_at(RunStatus::Blocked);
14597        runner.state.config.graph.selected_implementer = Some("selected".into());
14598        runner.state.save().unwrap();
14599        assert!(runner.execute().await.is_err());
14600        let disk = RunState::load(&runner.state.id).unwrap();
14601        assert!(disk.driver_exited);
14602        assert_eq!(disk.driver_pid, None);
14603        assert_eq!(disk.status, RunStatus::Blocked);
14604    }
14605
14606    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14607        crate::run::MergeRepair {
14608            question: "repair-question".into(),
14609            head: "old-head".into(),
14610            pr: "https://github.com/o/r/pull/42".into(),
14611            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14612            gate: vec![outcome(Some(0), false)],
14613            gate_head: Some("old-head".into()),
14614            gate_fixes: Vec::new(),
14615            pre_gate: Vec::new(),
14616            phase,
14617            failure: None,
14618        }
14619    }
14620
14621    #[tokio::test]
14622    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14623        crate::run::pin_test_home();
14624        let mut runner = runner_at(RunStatus::Landing);
14625        runner.state.merge_repairs.push(merge_repair_request(
14626            crate::run::MergeRepairPhase::Requested,
14627        ));
14628        runner.state.save().unwrap();
14629        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14630        assert!(
14631            runner
14632                .execute()
14633                .await
14634                .unwrap_err()
14635                .to_string()
14636                .contains("already running")
14637        );
14638        assert_eq!(
14639            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14640            crate::run::MergeRepairPhase::Requested
14641        );
14642        drop(claim);
14643        let mut disk = runner.state.clone();
14644        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14645        disk.save().unwrap();
14646        assert!(
14647            runner
14648                .execute()
14649                .await
14650                .unwrap_err()
14651                .to_string()
14652                .contains("decision changed")
14653        );
14654        assert_eq!(
14655            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14656            crate::run::MergeRepairPhase::Complete
14657        );
14658    }
14659
14660    #[test]
14661    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14662        crate::run::pin_test_home();
14663        let mut runner = runner_at(RunStatus::Landing);
14664        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14665        runner.state.reviews = request.reviews.clone();
14666        runner.state.gate = request.gate.clone();
14667        runner.state.gate_ran = true;
14668        runner.state.land_approval = Some(crate::run::LandApproval {
14669            question: "old".into(),
14670            head: "old-head".into(),
14671        });
14672        runner.state.merge_repairs.push(request);
14673        runner.begin_merge_repair_review(0).unwrap();
14674        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14675        assert!(!runner.state.gate_status().ok());
14676        assert!(runner.state.land_approval.is_none());
14677        let disk = RunState::load(&runner.state.id).unwrap();
14678        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14679        assert!(disk.merge_repairs[0].gate[0].ok());
14680        assert_eq!(
14681            disk.merge_repairs[0].phase,
14682            crate::run::MergeRepairPhase::Reviewing
14683        );
14684        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14685        assert!(disk.reviews.is_empty());
14686    }
14687
14688    #[tokio::test]
14689    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14690        crate::run::pin_test_home();
14691        for phase in [
14692            crate::run::MergeRepairPhase::Fixing,
14693            crate::run::MergeRepairPhase::Requested,
14694        ] {
14695            let mut runner = runner_at(RunStatus::Landing);
14696            runner.state.config.graph.land_rounds = 0;
14697            runner.state.merge_repairs.push(merge_repair_request(phase));
14698            runner.state.save().unwrap();
14699            assert!(!runner.repair_merge_review().await.unwrap());
14700            assert_eq!(runner.state.status, RunStatus::Blocked);
14701            assert_eq!(
14702                runner.state.merge_repairs[0].phase,
14703                crate::run::MergeRepairPhase::Failed
14704            );
14705            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14706            assert!(
14707                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14708                    "interrupted"
14709                } else {
14710                    "budget exhausted"
14711                })
14712            );
14713            assert!(runner.state.merge.is_none());
14714        }
14715    }
14716
14717    /// `park_here` folding in the reason `Pause::park_because` recorded -
14718    /// this is what lets an operator reading a run's events tell an
14719    /// interrupt-driven park from an ordinary shutdown park.
14720    #[test]
14721    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14722        crate::run::pin_test_home();
14723        let mut runner = runner_at(RunStatus::Implementing);
14724        let interrupt = Pause::new();
14725        runner.watch_interrupt(interrupt.clone());
14726
14727        interrupt.park_because("task a1b2 asked to run first");
14728
14729        assert!(runner.park_here().expect("park_here"));
14730        assert!(runner.state.parked);
14731        let last = runner.state.events.last().expect("a park event");
14732        assert_eq!(last.node, "park");
14733        assert!(
14734            last.message.contains("task a1b2 asked to run first"),
14735            "expected the interrupt reason in {:?}",
14736            last.message
14737        );
14738    }
14739
14740    /// The cause is stored on the pass when it parks, and the first cause a
14741    /// shared pause was given is the one that stays.
14742    #[test]
14743    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14744        crate::run::pin_test_home();
14745        let mut runner = runner_at(RunStatus::Implementing);
14746        runner.state.begin_pass();
14747        let shutdown = Pause::new();
14748        runner.on_pause(shutdown.clone());
14749        let upgrade = ParkCause::Upgrade {
14750            from: "0.121.0".to_owned(),
14751            to: Some("0.121.1".to_owned()),
14752        };
14753        shutdown.park_for(upgrade.clone());
14754        shutdown.park_for(ParkCause::Operator);
14755
14756        assert!(runner.park_here().expect("park_here"));
14757        assert_eq!(
14758            runner.state.passes[0].end,
14759            Some(crate::run::PassEnd::Parked { cause: upgrade })
14760        );
14761        let last = runner.state.events.last().expect("a park event");
14762        assert!(last.message.contains("0.121.0"), "{}", last.message);
14763    }
14764
14765    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
14766    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
14767    /// cleared) must not make a *different* run - one only watching its own,
14768    /// unshared interrupt `Pause` - see itself as parked. If a future change
14769    /// ever collapsed these back into one handle, the interrupt scheduler
14770    /// would park every run for the rest of the daemon's life, not just the
14771    /// one it meant to interrupt.
14772    #[test]
14773    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14774        crate::run::pin_test_home();
14775        let mut runner = runner_at(RunStatus::Implementing);
14776        let shutdown = Pause::new();
14777        runner.on_pause(shutdown.clone());
14778        let interrupt = Pause::new();
14779        runner.watch_interrupt(interrupt.clone());
14780
14781        // Nobody has asked for anything yet.
14782        assert!(!runner.park_here().expect("park_here"));
14783        assert!(!runner.state.parked);
14784
14785        // Only the interrupt handle fires; the shutdown handle stays clear.
14786        interrupt.park_because("test");
14787        assert!(!shutdown.parked());
14788        assert!(runner.park_here().expect("park_here"));
14789    }
14790
14791    /// The property every prior attempt at this feature failed to pin down:
14792    /// asking a run to park while one of its nodes has a real, in-flight
14793    /// async operation running (an agent call, in production) must not cut
14794    /// that operation short. `park_here` is only ever consulted *between*
14795    /// `execute`'s node calls - see its own doc - so nothing inside a node
14796    /// can observe a park request until the node itself returns. This proves
14797    /// that structurally, with real `tokio` concurrency and a channel
14798    /// handshake (never a sleep, which would only prove "usually", not
14799    /// "cannot"): the "node" below reports that it has genuinely started,
14800    /// and only then is the park requested; the node still has to be told to
14801    /// finish before `park_here` is ever called, exactly mirroring every
14802    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
14803    /// in `execute`.
14804    #[tokio::test]
14805    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14806        crate::run::pin_test_home();
14807        let mut runner = runner_at(RunStatus::Implementing);
14808        let interrupt = Pause::new();
14809        runner.watch_interrupt(interrupt.clone());
14810
14811        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14812        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14813
14814        // Stands in for one node's in-flight agent call: it proves it has
14815        // genuinely started, then blocks - exactly as a spawned CLI process
14816        // does - until told to finish.
14817        let node = async move {
14818            started_tx.send(()).expect("send started");
14819            finish_rx.await.expect("recv finish");
14820            "node finished"
14821        };
14822
14823        let interrupter = async move {
14824            started_rx.await.expect("recv started");
14825            // The call is now genuinely in flight. Ask it to park.
14826            interrupt.park_because("higher-priority task waiting");
14827            // Nothing the node does can observe this yet - there is no
14828            // check inside it, by construction - so let the executor run
14829            // anything pending and then let the node finish on its own.
14830            tokio::task::yield_now().await;
14831            finish_tx.send(()).expect("send finish");
14832        };
14833
14834        let (node_result, ()) = tokio::join!(node, interrupter);
14835        assert_eq!(
14836            node_result, "node finished",
14837            "the in-flight call ran to completion"
14838        );
14839
14840        // Only now, at the boundary the real `execute` would check right
14841        // after this node, does the park take effect.
14842        assert!(runner.park_here().expect("park_here"));
14843        assert!(runner.state.parked);
14844    }
14845
14846    /// A run parked mid-competition carries every field it had accumulated
14847    /// through the exact same disk round-trip an ordinary resume uses -
14848    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
14849    /// Nothing about parking for an interrupt is a special case of that path;
14850    /// this is what proves it rather than assuming it.
14851    #[test]
14852    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14853        crate::run::pin_test_home();
14854        let mut runner = runner_at(RunStatus::Judging);
14855        // `Runner::resume` re-resolves roles from the saved config, which
14856        // refuses an empty roster - give it the same minimal one `conductor`
14857        // itself uses.
14858        runner.state.config.agents = vec![conductor()];
14859        runner.state.candidates = vec![Candidate {
14860            index: 0,
14861            label: 'A',
14862            agent: "alpha".to_owned(),
14863            branch: "magi/x/A".to_owned(),
14864            worktree: PathBuf::from("/nonexistent/worktree"),
14865            summary: "did the thing".to_owned(),
14866            stat: "1 file changed".to_owned(),
14867            files: 1,
14868            commits: 1,
14869            empty: false,
14870            failed: None,
14871            verified_noop: None,
14872            duration_ms: 1234,
14873            folded: false,
14874        }];
14875        let run_id = runner.state.id.clone();
14876
14877        let interrupt = Pause::new();
14878        runner.watch_interrupt(interrupt.clone());
14879        interrupt.park_because("task c3d4 asked to run first");
14880        assert!(runner.park_here().expect("park_here"));
14881
14882        let resumed = Runner::resume(&run_id).expect("resume");
14883        assert_eq!(resumed.state.candidates.len(), 1);
14884        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14885        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14886        assert_eq!(resumed.state.status, runner.state.status);
14887        assert!(
14888            resumed.state.parked,
14889            "still parked until `execute` actually walks the graph again"
14890        );
14891        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14892    }
14893
14894    /// A fresh open question on `run`, stored and handed back for assertions.
14895    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14896        let mut q = ask::Question::new(
14897            run.to_owned(),
14898            "implement".to_owned(),
14899            "impl-A".to_owned(),
14900            "Which storage backend should the cache use?".to_owned(),
14901            String::new(),
14902            vec!["SQLite".to_owned(), "Redis".to_owned()],
14903        );
14904        store.put(&mut q).unwrap();
14905        q
14906    }
14907
14908    #[test]
14909    fn a_failed_runs_open_question_is_abandoned() {
14910        ask_test_home();
14911        let store = ask::Questions::open();
14912        let mut runner = runner_at(RunStatus::Failed);
14913        let run = runner.state.id.clone();
14914        let q = ask_open_question(&store, &run);
14915
14916        runner.settle_questions();
14917
14918        let back = store.get(&q.id).unwrap();
14919        assert!(
14920            !back.status.open(),
14921            "the seat that asked died with the run; nobody is left to read an answer"
14922        );
14923        assert!(
14924            back.detail.contains(&run) && back.detail.contains("failed"),
14925            "the reason names what the run became, not just that it is gone: {}",
14926            back.detail
14927        );
14928    }
14929
14930    #[test]
14931    fn a_merged_runs_open_question_is_abandoned_too() {
14932        ask_test_home();
14933        let store = ask::Questions::open();
14934        // A run that finishes cleanly still leaves nobody to read an answer -
14935        // this is not only a failure-path cleanup.
14936        for status in [RunStatus::Merged, RunStatus::Ready] {
14937            let mut runner = runner_at(status);
14938            let run = runner.state.id.clone();
14939            let q = ask_open_question(&store, &run);
14940
14941            runner.settle_questions();
14942
14943            let back = store.get(&q.id).unwrap();
14944            assert!(
14945                !back.status.open(),
14946                "{status:?} run's question must not outlive the run"
14947            );
14948        }
14949    }
14950
14951    #[test]
14952    fn a_still_resumable_runs_open_question_is_left_alone() {
14953        ask_test_home();
14954        let store = ask::Questions::open();
14955        // `Blocked` and `Stalled` can still be resumed — the candidates, the
14956        // review round and the seat sessions are all still on disk — so a
14957        // question asked mid-round may yet get a real answer from a real
14958        // resume. Sweeping it here would be exactly the failure mode this
14959        // whole feature exists to avoid on the other side.
14960        for status in [RunStatus::Blocked, RunStatus::Stalled] {
14961            let mut runner = runner_at(status);
14962            let run = runner.state.id.clone();
14963            let q = ask_open_question(&store, &run);
14964
14965            runner.settle_questions();
14966
14967            let back = store.get(&q.id).unwrap();
14968            assert!(
14969                back.status.open(),
14970                "{status:?} is still alive; the question must still be waiting"
14971            );
14972        }
14973    }
14974
14975    #[test]
14976    fn settle_questions_never_touches_an_already_answered_question() {
14977        ask_test_home();
14978        let store = ask::Questions::open();
14979        let mut runner = runner_at(RunStatus::Failed);
14980        let run = runner.state.id.clone();
14981        let mut q = ask_open_question(&store, &run);
14982        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14983            .unwrap();
14984        store.put(&mut q).unwrap();
14985
14986        // Called twice, the way a crash-recovered daemon reclaim and the
14987        // graph's own cleanup both can for the same run — `abandon_for_run`
14988        // only ever touches what is still open, so this must be inert both
14989        // times, not merely the second.
14990        runner.settle_questions();
14991        runner.settle_questions();
14992
14993        let back = store.get(&q.id).unwrap();
14994        assert_eq!(
14995            back.status,
14996            ask::QuestionStatus::Answered,
14997            "a real answer is a decision on record, never overwritten by a sweep"
14998        );
14999    }
15000
15001    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
15002    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
15003    /// without merging, whose winner is still the operator's answer to read.
15004    /// Nothing previously called `fold_run` itself with a real `tally`, so
15005    /// this is the first test to pin down the one distinction the whole
15006    /// automatic-fold feature depends on: the winner's worktree and branch
15007    /// must survive, everything else sharing the run's worktree bay - a
15008    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
15009    /// second sweep treats every non-winner directory under the bay alike -
15010    /// must not.
15011    #[tokio::test]
15012    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15013        crate::run::pin_test_home();
15014        let tmp = tempfile::tempdir().expect("tempdir");
15015        let repo = tmp.path().join("repo");
15016        std::fs::create_dir_all(&repo).unwrap();
15017        init_repo(&repo);
15018
15019        let mut config = Config::default();
15020        config.graph.worktree_root = Some(tmp.path().join("wt"));
15021
15022        let mut state = RunState::new(
15023            repo.clone(),
15024            "main".to_owned(),
15025            "deadbeef".to_owned(),
15026            "task".to_owned(),
15027            config,
15028        );
15029        let root = state.worktree_root();
15030        let wt_a = root.join("cand-A");
15031        let wt_b = root.join("cand-B");
15032        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15033            .await
15034            .expect("worktree A");
15035        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15036            .await
15037            .expect("worktree B");
15038
15039        state.candidates = vec![
15040            Candidate {
15041                index: 0,
15042                label: 'A',
15043                agent: "alpha".to_owned(),
15044                branch: "magi/x/A".to_owned(),
15045                worktree: wt_a.clone(),
15046                summary: String::new(),
15047                stat: String::new(),
15048                files: 0,
15049                commits: 0,
15050                empty: false,
15051                failed: None,
15052                verified_noop: None,
15053                duration_ms: 0,
15054                folded: false,
15055            },
15056            Candidate {
15057                index: 1,
15058                label: 'B',
15059                agent: "beta".to_owned(),
15060                branch: "magi/x/B".to_owned(),
15061                worktree: wt_b.clone(),
15062                summary: String::new(),
15063                stat: String::new(),
15064                files: 0,
15065                commits: 0,
15066                empty: false,
15067                failed: None,
15068                verified_noop: None,
15069                duration_ms: 0,
15070                folded: false,
15071            },
15072        ];
15073        state.tally = Some(Tally {
15074            first_choice: BTreeMap::from([('A', 1)]),
15075            borda: BTreeMap::new(),
15076            winner: 'A',
15077            rankings: 1,
15078            unanimous_initial: true,
15079            deliberated: false,
15080            changed_votes: 0,
15081            unanimous_final: true,
15082            tie_break: None,
15083            judges: 1,
15084            present: 1,
15085            quorum: 1,
15086            met_quorum: true,
15087            uncontested: None,
15088        });
15089        state.status = RunStatus::Ready;
15090
15091        fold_run(&mut state, false, &crate::run::home())
15092            .await
15093            .expect("fold_run");
15094
15095        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15096        assert!(
15097            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15098            "the unmerged winner's branch survives"
15099        );
15100        assert!(
15101            !state.candidates[0].folded,
15102            "the winner is not marked folded"
15103        );
15104
15105        assert!(!wt_b.exists(), "the loser's worktree is removed");
15106        assert!(
15107            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15108            "the loser's branch is removed"
15109        );
15110        assert!(state.candidates[1].folded, "the loser is marked folded");
15111    }
15112
15113    #[tokio::test]
15114    #[cfg(unix)]
15115    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15116        use std::os::fd::AsRawFd as _;
15117        use std::time::Duration;
15118
15119        let tmp = tempfile::tempdir().unwrap();
15120        let repo = tmp.path().join("repo");
15121        std::fs::create_dir_all(&repo).unwrap();
15122        init_repo(&repo);
15123        let common = repo.join(".git");
15124        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15125        let home = tmp.path().join("home");
15126        let mut config = Config::default();
15127        config.coordination = crate::config::test_coordination();
15128        config.graph.worktree_root = Some(tmp.path().join("wt"));
15129        config.graph.timeout_resource_wait = 1;
15130        let mut state = RunState::new(
15131            repo.clone(),
15132            "main".into(),
15133            "deadbeef".into(),
15134            "task".into(),
15135            config,
15136        );
15137        let root = state.worktree_root();
15138        let candidate = root.join("cand-A");
15139        let reviewer = root.join("review-1");
15140        let branch = state.branch_for('A');
15141        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15142            .await
15143            .unwrap();
15144        git::worktree_add_detached(&repo, &reviewer, "main")
15145            .await
15146            .unwrap();
15147        git::acquire_worktree_config(&repo).await.unwrap();
15148        state.enabled_worktree_config = true;
15149        state.status = RunStatus::Ready;
15150        state.candidates.push(Candidate {
15151            index: 0,
15152            label: 'A',
15153            agent: "implement".into(),
15154            branch: branch.clone(),
15155            worktree: candidate.clone(),
15156            summary: String::new(),
15157            stat: String::new(),
15158            files: 0,
15159            commits: 0,
15160            empty: false,
15161            failed: None,
15162            verified_noop: None,
15163            duration_ms: 0,
15164            folded: false,
15165        });
15166        state.save_under(&home).unwrap();
15167        let config_before = std::fs::read(common.join("config")).unwrap();
15168        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15169            .await
15170            .unwrap();
15171        // A separate OS handle bypasses the launcher's local gate, exactly as
15172        // an external updater does, and must still prevent all folding work.
15173        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15174        assert_eq!(
15175            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15176            0
15177        );
15178        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15179        assert!(
15180            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15181                .await
15182                .is_err()
15183        );
15184        assert!(candidate.exists());
15185        assert!(reviewer.exists());
15186        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15187        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15188        assert_eq!(
15189            git::git(&repo, &["worktree", "list", "--porcelain"])
15190                .await
15191                .unwrap(),
15192            worktrees_before
15193        );
15194        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15195            .await
15196            .unwrap()
15197            .unwrap_err();
15198        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15199        drop(fold);
15200        assert!(candidate.exists() && reviewer.exists());
15201        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15202        assert_eq!(
15203            state.resource_block.as_ref().unwrap().kind,
15204            "repository coordination"
15205        );
15206        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15207        drop(external);
15208        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15209        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15210            .await
15211            .unwrap()
15212            .unwrap();
15213        drop(fold);
15214        assert!(removed.contains(&branch));
15215        assert!(!root.exists());
15216        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15217        assert!(state.candidates[0].folded);
15218        assert!(!state.enabled_worktree_config);
15219        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15220        let enabled = git::git_raw(
15221            &repo,
15222            &["config", "--local", "--get", "extensions.worktreeConfig"],
15223        )
15224        .await
15225        .unwrap();
15226        assert!(!enabled.ok(), "the last config reference was released");
15227    }
15228
15229    /// A branch handed to a later run is that run's (and its pull request's):
15230    /// folding the run that released it must not delete it.
15231    #[tokio::test]
15232    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15233        let tmp = tempfile::tempdir().expect("tempdir");
15234        let repo = tmp.path().join("repo");
15235        std::fs::create_dir_all(&repo).unwrap();
15236        init_repo(&repo);
15237        let home = tmp.path().join("home");
15238
15239        let mut config = Config::default();
15240        config.graph.worktree_root = Some(tmp.path().join("wt"));
15241        let mut state = RunState::new(
15242            repo.clone(),
15243            "main".to_owned(),
15244            "deadbeef".to_owned(),
15245            "task".to_owned(),
15246            config,
15247        );
15248        // The worktree is already gone (released); the branch survives.
15249        git::git(&repo, &["branch", "magi/x/A", "main"])
15250            .await
15251            .expect("branch");
15252        state.candidates = vec![Candidate {
15253            index: 0,
15254            label: 'A',
15255            agent: "alpha".to_owned(),
15256            branch: "magi/x/A".to_owned(),
15257            worktree: state.worktree_root().join("cand-A"),
15258            summary: String::new(),
15259            stat: String::new(),
15260            files: 0,
15261            commits: 0,
15262            empty: false,
15263            failed: None,
15264            verified_noop: None,
15265            duration_ms: 0,
15266            folded: true,
15267        }];
15268        state.released_to = Some("20260901-000000-new1".to_owned());
15269        state.released_branches = vec!["magi/x/A".to_owned()];
15270
15271        fold_run(&mut state, true, &home).await.expect("fold_run");
15272
15273        assert!(
15274            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15275            "the handed-over branch survives a fold"
15276        );
15277    }
15278
15279    /// A winner with nothing ahead of the base is caught before `gh` is ever
15280    /// asked for a pull request, and the message carries what the task's
15281    /// references resolved to.
15282    #[tokio::test]
15283    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15284        let tmp = tempfile::tempdir().expect("tempdir");
15285        let repo = tmp.path().join("repo");
15286        std::fs::create_dir_all(&repo).unwrap();
15287        init_repo(&repo);
15288        let run = |args: &[&str]| {
15289            let out = std::process::Command::new("git")
15290                .quiet()
15291                .args(args)
15292                .current_dir(&repo)
15293                .output()
15294                .expect("spawn git");
15295            assert!(out.status.success(), "git {args:?}");
15296        };
15297        run(&["branch", "magi/x/A"]);
15298        run(&["checkout", "-q", "-b", "magi/x/B"]);
15299        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15300        run(&["add", "-A"]);
15301        run(&["commit", "-q", "-m", "work"]);
15302        run(&["checkout", "-q", "main"]);
15303
15304        // A pull request is compared against the remote's base, so the
15305        // fixture needs one. Before it exists nothing can be read, and the
15306        // answer must be "not empty".
15307        let probe = RunState::new(
15308            repo.clone(),
15309            "main".to_owned(),
15310            "deadbeef".to_owned(),
15311            "task".to_owned(),
15312            Config::default(),
15313        );
15314        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15315        let bare = tmp.path().join("origin.git");
15316        let out = std::process::Command::new("git")
15317            .quiet()
15318            .args(["init", "-q", "--bare"])
15319            .arg(&bare)
15320            .output()
15321            .expect("spawn git");
15322        assert!(out.status.success(), "git init --bare");
15323        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15324        run(&["push", "-q", "origin", "main"]);
15325
15326        let mut state = RunState::new(
15327            repo.clone(),
15328            "main".to_owned(),
15329            "deadbeef".to_owned(),
15330            "task".to_owned(),
15331            Config::default(),
15332        );
15333        state.seeds = vec![refs::Seed {
15334            token: "magi/27b2/A".to_owned(),
15335            kind: refs::SeedKind::Unresolved,
15336            sha: String::new(),
15337            branch: true,
15338            detail: "no branch or commit named magi/27b2/A".to_owned(),
15339        }];
15340
15341        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15342        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15343        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15344        let detail = empty_candidate_detail(&state, "main");
15345        assert!(detail.starts_with("empty candidate"), "{detail}");
15346        assert!(detail.contains("magi/27b2/A"), "{detail}");
15347    }
15348
15349    /// `status == Ready` used to be read as "this is the harmless
15350    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15351    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15352    /// run's PR was closed without merging — and reentering `merge` with
15353    /// `mode` still `Pr` does not know the difference, so it pushed and
15354    /// opened a second pull request. `mode == Local` reproduces the same
15355    /// blind spot without a network call: reentry must not attempt another
15356    /// git merge once this node has already recorded an outcome.
15357    #[tokio::test]
15358    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15359        let tmp = tempfile::tempdir().expect("tempdir");
15360        let repo = tmp.path().join("repo");
15361        std::fs::create_dir_all(&repo).unwrap();
15362        init_repo(&repo);
15363
15364        let mut config = Config::default();
15365        config.merge.mode = MergeMode::Local;
15366
15367        let mut state = RunState::new(
15368            repo.clone(),
15369            "main".to_owned(),
15370            "deadbeef".to_owned(),
15371            "task".to_owned(),
15372            config,
15373        );
15374        state.candidates = vec![Candidate {
15375            index: 0,
15376            label: 'A',
15377            agent: "alpha".to_owned(),
15378            branch: "does-not-exist".to_owned(),
15379            worktree: repo.clone(),
15380            summary: String::new(),
15381            stat: String::new(),
15382            files: 0,
15383            commits: 0,
15384            empty: false,
15385            failed: None,
15386            verified_noop: None,
15387            duration_ms: 0,
15388            folded: false,
15389        }];
15390        state.tally = Some(Tally {
15391            first_choice: BTreeMap::from([('A', 1)]),
15392            borda: BTreeMap::new(),
15393            winner: 'A',
15394            rankings: 1,
15395            unanimous_initial: true,
15396            deliberated: false,
15397            changed_votes: 0,
15398            unanimous_final: true,
15399            tie_break: None,
15400            judges: 0,
15401            present: 0,
15402            quorum: 0,
15403            met_quorum: true,
15404            uncontested: Some("only candidate A produced a change".to_owned()),
15405        });
15406        state.reviews = vec![ReviewRound {
15407            round: 1,
15408            head: "deadbeef".to_owned(),
15409            verified_head: None,
15410            verified_at: None,
15411            reviews: Vec::new(),
15412            e2e: Vec::new(),
15413            fix: None,
15414            blocking: 0,
15415            answered: 0,
15416            expected: 0,
15417            clean: true,
15418            verify_retried: false,
15419            e2e_deferred: false,
15420            e2e_defer_reason: None,
15421            progressed: false,
15422            vote_split: false,
15423            reconsideration: Vec::new(),
15424            verdict: None,
15425        }];
15426        state.gate = vec![CommandOutcome {
15427            command: "test".to_owned(),
15428            code: Some(0),
15429            output_tail: String::new(),
15430            duration_ms: 0,
15431            resource_blocked: false,
15432        }];
15433        state.gate_ran = true;
15434        // Reached its conclusion already — e.g. `land` closing the PR without
15435        // merging it, which (like the honest `MergeMode::None` path) leaves
15436        // `status` at `Ready`. The recorded outcome is what actually marks
15437        // this node done.
15438        state.status = RunStatus::Ready;
15439        state.merge = Some(MergeOutcome {
15440            mode: MergeMode::Local,
15441            ok: false,
15442            detail: "already concluded".to_owned(),
15443            empty: false,
15444        });
15445
15446        let mut runner = Runner {
15447            state,
15448            roles: ResolvedRoles {
15449                implementers: Vec::new(),
15450                judges: Vec::new(),
15451                reviewers: Vec::new(),
15452                fixer: None,
15453                conductor: conductor(),
15454                implementer_roster: Vec::new(),
15455                judge_roster: Vec::new(),
15456                reviewer_roster: Vec::new(),
15457            },
15458            sem: Arc::new(Semaphore::new(1)),
15459            pause: Pause::new(),
15460            interrupt: Pause::new(),
15461        };
15462
15463        runner.merge().await.expect("merge");
15464
15465        assert_eq!(
15466            runner.state.status,
15467            RunStatus::Ready,
15468            "a concluded run's status must not change on reentry"
15469        );
15470        assert_eq!(
15471            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15472            Some("already concluded"),
15473            "merge must not run again once the node already recorded an outcome"
15474        );
15475    }
15476
15477    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15478    /// when its last attempt was resource-blocked (the shared build cache
15479    /// could not be acquired or confirmed fresh in time - see
15480    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15481    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15482    /// a run merge a tree the gate never actually checked - exactly the case
15483    /// a contended cache produces on every retry until it clears. `merge`
15484    /// must refuse until `gate` has actually recorded an attempt.
15485    #[tokio::test]
15486    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15487        let tmp = tempfile::tempdir().expect("tempdir");
15488        let repo = tmp.path().join("repo");
15489        std::fs::create_dir_all(&repo).unwrap();
15490        init_repo(&repo);
15491
15492        let mut config = Config::default();
15493        config.merge.mode = MergeMode::Local;
15494
15495        let mut state = RunState::new(
15496            repo.clone(),
15497            "main".to_owned(),
15498            "deadbeef".to_owned(),
15499            "task".to_owned(),
15500            config,
15501        );
15502        state.candidates = vec![Candidate {
15503            index: 0,
15504            label: 'A',
15505            agent: "alpha".to_owned(),
15506            branch: "does-not-exist".to_owned(),
15507            worktree: repo.clone(),
15508            summary: String::new(),
15509            stat: String::new(),
15510            files: 0,
15511            commits: 0,
15512            empty: false,
15513            failed: None,
15514            verified_noop: None,
15515            duration_ms: 0,
15516            folded: false,
15517        }];
15518        state.tally = Some(Tally {
15519            first_choice: BTreeMap::from([('A', 1)]),
15520            borda: BTreeMap::new(),
15521            winner: 'A',
15522            rankings: 1,
15523            unanimous_initial: true,
15524            deliberated: false,
15525            changed_votes: 0,
15526            unanimous_final: true,
15527            tie_break: None,
15528            judges: 0,
15529            present: 0,
15530            quorum: 0,
15531            met_quorum: true,
15532            uncontested: Some("only candidate A produced a change".to_owned()),
15533        });
15534        state.reviews = vec![ReviewRound {
15535            round: 1,
15536            head: "deadbeef".to_owned(),
15537            verified_head: None,
15538            verified_at: None,
15539            reviews: Vec::new(),
15540            e2e: Vec::new(),
15541            fix: None,
15542            blocking: 0,
15543            answered: 0,
15544            expected: 0,
15545            clean: true,
15546            verify_retried: false,
15547            e2e_deferred: false,
15548            e2e_defer_reason: None,
15549            progressed: false,
15550            vote_split: false,
15551            reconsideration: Vec::new(),
15552            verdict: None,
15553        }];
15554        // The point: `gate` has not recorded anything yet.
15555        state.gate = Vec::new();
15556        state.gate_ran = false;
15557        state.status = RunStatus::Gating;
15558
15559        let mut runner = Runner {
15560            state,
15561            roles: ResolvedRoles {
15562                implementers: Vec::new(),
15563                judges: Vec::new(),
15564                reviewers: Vec::new(),
15565                fixer: None,
15566                conductor: conductor(),
15567                implementer_roster: Vec::new(),
15568                judge_roster: Vec::new(),
15569                reviewer_roster: Vec::new(),
15570            },
15571            sem: Arc::new(Semaphore::new(1)),
15572            pause: Pause::new(),
15573            interrupt: Pause::new(),
15574        };
15575
15576        runner.merge().await.expect("merge");
15577
15578        assert!(
15579            runner.state.merge.is_none(),
15580            "an empty gate must never be read as a passing one: {:?}",
15581            runner.state.merge
15582        );
15583    }
15584
15585    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
15586    /// commands configured and `merge.mode` is `none` (a review-only run).
15587    /// `gate` must still record a real attempt — zero commands, vacuously
15588    /// passed — rather than leaving `state.gate` empty in a way `merge`
15589    /// cannot tell apart from "never ran"; otherwise the run reaches
15590    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
15591    #[tokio::test]
15592    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15593        let tmp = tempfile::tempdir().expect("tempdir");
15594        let repo = tmp.path().join("repo");
15595        std::fs::create_dir_all(&repo).unwrap();
15596        init_repo(&repo);
15597
15598        // Default config: `verify.gate` empty, `merge.mode` is `none`.
15599        let config = Config::default();
15600
15601        let mut state = RunState::new(
15602            repo.clone(),
15603            "main".to_owned(),
15604            "deadbeef".to_owned(),
15605            "task".to_owned(),
15606            config,
15607        );
15608        state.candidates = vec![Candidate {
15609            index: 0,
15610            label: 'A',
15611            agent: "alpha".to_owned(),
15612            branch: "does-not-exist".to_owned(),
15613            worktree: repo.clone(),
15614            summary: String::new(),
15615            stat: String::new(),
15616            files: 0,
15617            commits: 0,
15618            empty: false,
15619            failed: None,
15620            verified_noop: None,
15621            duration_ms: 0,
15622            folded: false,
15623        }];
15624        state.tally = Some(Tally {
15625            first_choice: BTreeMap::from([('A', 1)]),
15626            borda: BTreeMap::new(),
15627            winner: 'A',
15628            rankings: 1,
15629            unanimous_initial: true,
15630            deliberated: false,
15631            changed_votes: 0,
15632            unanimous_final: true,
15633            tie_break: None,
15634            judges: 0,
15635            present: 0,
15636            quorum: 0,
15637            met_quorum: true,
15638            uncontested: Some("only candidate A produced a change".to_owned()),
15639        });
15640        state.reviews = vec![ReviewRound {
15641            round: 1,
15642            head: "deadbeef".to_owned(),
15643            verified_head: None,
15644            verified_at: None,
15645            reviews: Vec::new(),
15646            e2e: Vec::new(),
15647            fix: None,
15648            blocking: 0,
15649            answered: 0,
15650            expected: 0,
15651            clean: true,
15652            verify_retried: false,
15653            e2e_deferred: false,
15654            e2e_defer_reason: None,
15655            progressed: false,
15656            vote_split: false,
15657            reconsideration: Vec::new(),
15658            verdict: None,
15659        }];
15660
15661        let mut runner = Runner {
15662            state,
15663            roles: ResolvedRoles {
15664                implementers: Vec::new(),
15665                judges: Vec::new(),
15666                reviewers: Vec::new(),
15667                fixer: None,
15668                conductor: conductor(),
15669                implementer_roster: Vec::new(),
15670                judge_roster: Vec::new(),
15671                reviewer_roster: Vec::new(),
15672            },
15673            sem: Arc::new(Semaphore::new(1)),
15674            pause: Pause::new(),
15675            interrupt: Pause::new(),
15676        };
15677
15678        runner.gate().await.expect("gate");
15679        assert!(
15680            runner.state.gate_ran,
15681            "zero configured commands is still a real attempt, not an unrun gate"
15682        );
15683        assert!(runner.state.gate.is_empty());
15684        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15685        assert_ne!(
15686            runner.state.status,
15687            RunStatus::Blocked,
15688            "a gate with nothing to check must not read as failed"
15689        );
15690
15691        runner.merge().await.expect("merge");
15692        assert_eq!(
15693            runner.state.status,
15694            RunStatus::Ready,
15695            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15696        );
15697    }
15698
15699    /// `Config::cache_dir` is derived from `verify.e2e` as well as
15700    /// `verify.gate` (so the e2e leg and the final gate never build against
15701    /// different directories). With zero `verify.gate` commands but a
15702    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
15703    /// that lease before discovering it had nothing to run - so a repo with
15704    /// no gate commands could come back `resource_blocked` (and therefore
15705    /// still `gate_ran == false`) on nothing but an unrelated run holding the
15706    /// cache, exactly the contention this run's own zero commands could
15707    /// never have touched. `gate` must recognise there is nothing to check
15708    /// before it ever asks for the lease.
15709    #[tokio::test]
15710    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15711        crate::run::pin_test_home();
15712        let home = crate::run::home();
15713
15714        let tmp = tempfile::tempdir().expect("tempdir");
15715        let repo = tmp.path().join("repo");
15716        std::fs::create_dir_all(&repo).unwrap();
15717        init_repo(&repo);
15718        // Unique to this test, so holding its lease cannot collide with
15719        // another test sharing the same process-wide `home`.
15720        let cache_dir = tmp.path().join("target");
15721
15722        let mut config = Config::default();
15723        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15724        // `verify.gate` stays empty (the default). Bounded so a regression
15725        // that does start waiting fails the test in seconds, not hangs it.
15726        config.graph.timeout_verify = Some(2);
15727        config.graph.timeout_resource_wait = 2;
15728
15729        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15730        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15731            .expect("no io error acquiring directly")
15732        {
15733            crate::cache::AcquireOutcome::Acquired(g) => g,
15734            crate::cache::AcquireOutcome::Busy(b) => {
15735                panic!("expected the direct acquire to win the lease first: {b:?}")
15736            }
15737        };
15738
15739        let mut state = RunState::new(
15740            repo.clone(),
15741            "main".to_owned(),
15742            "deadbeef".to_owned(),
15743            "task".to_owned(),
15744            config,
15745        );
15746        state.candidates = vec![Candidate {
15747            index: 0,
15748            label: 'A',
15749            agent: "alpha".to_owned(),
15750            branch: "does-not-exist".to_owned(),
15751            worktree: repo.clone(),
15752            summary: String::new(),
15753            stat: String::new(),
15754            files: 0,
15755            commits: 0,
15756            empty: false,
15757            failed: None,
15758            verified_noop: None,
15759            duration_ms: 0,
15760            folded: false,
15761        }];
15762        state.tally = Some(Tally {
15763            first_choice: BTreeMap::from([('A', 1)]),
15764            borda: BTreeMap::new(),
15765            winner: 'A',
15766            rankings: 1,
15767            unanimous_initial: true,
15768            deliberated: false,
15769            changed_votes: 0,
15770            unanimous_final: true,
15771            tie_break: None,
15772            judges: 0,
15773            present: 0,
15774            quorum: 0,
15775            met_quorum: true,
15776            uncontested: Some("only candidate A produced a change".to_owned()),
15777        });
15778        state.reviews = vec![ReviewRound {
15779            round: 1,
15780            head: "deadbeef".to_owned(),
15781            verified_head: None,
15782            verified_at: None,
15783            reviews: Vec::new(),
15784            e2e: Vec::new(),
15785            fix: None,
15786            blocking: 0,
15787            answered: 0,
15788            expected: 0,
15789            clean: true,
15790            verify_retried: false,
15791            e2e_deferred: false,
15792            e2e_defer_reason: None,
15793            progressed: false,
15794            vote_split: false,
15795            reconsideration: Vec::new(),
15796            verdict: None,
15797        }];
15798
15799        let mut runner = Runner {
15800            state,
15801            roles: ResolvedRoles {
15802                implementers: Vec::new(),
15803                judges: Vec::new(),
15804                reviewers: Vec::new(),
15805                fixer: None,
15806                conductor: conductor(),
15807                implementer_roster: Vec::new(),
15808                judge_roster: Vec::new(),
15809                reviewer_roster: Vec::new(),
15810            },
15811            sem: Arc::new(Semaphore::new(1)),
15812            pause: Pause::new(),
15813            interrupt: Pause::new(),
15814        };
15815
15816        let started = std::time::Instant::now();
15817        runner.gate().await.expect("gate");
15818        assert!(
15819            started.elapsed() < Duration::from_secs(1),
15820            "a gate with nothing to run must never wait on a lease it never needed"
15821        );
15822        assert!(
15823            runner.state.gate_ran,
15824            "zero commands is still a real, immediate attempt"
15825        );
15826        assert!(runner.state.gate.is_empty());
15827        assert_ne!(
15828            runner.state.status,
15829            RunStatus::Blocked,
15830            "must not read as resource-blocked on a lease it never asked for"
15831        );
15832    }
15833
15834    #[cfg(unix)]
15835    #[tokio::test]
15836    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15837        crate::run::pin_test_home();
15838        let tmp = tempfile::tempdir().unwrap();
15839        let mut config = Config::default();
15840        config.disk.min_free_bytes = 0;
15841        let mut state = RunState::new(
15842            tmp.path().to_owned(),
15843            "main".into(),
15844            "base".into(),
15845            "verification evidence".into(),
15846            config,
15847        );
15848        let (outcomes, pids) = run_commands(
15849            &mut state,
15850            "gate",
15851            "gate",
15852            1,
15853            &["sh".into(), "-c".into()],
15854            &[
15855                "printf before-timeout; exec sleep 30".into(),
15856                "kill -TERM $$".into(),
15857                "exit 7".into(),
15858                "printf success".into(),
15859            ],
15860            tmp.path(),
15861            Duration::from_secs(1),
15862        )
15863        .await;
15864        wait_for_timed_out_children_to_die(&pids).await;
15865        assert_eq!(
15866            pids.len(),
15867            1,
15868            "unleased timeout retains its death-check PID"
15869        );
15870        assert_eq!(outcomes.len(), 4);
15871        assert_eq!(outcomes[0].code, None);
15872        assert!(outcomes[0].output_tail.contains("before-timeout"));
15873        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15874        assert_eq!(outcomes[1].code, None);
15875        assert!(outcomes[1].output_tail.contains("command terminated:"));
15876        assert!(!outcomes[1].output_tail.contains("timed out"));
15877        assert_eq!(outcomes[2].code, Some(7));
15878        assert!(!outcomes[2].output_tail.contains("timed out"));
15879        assert_eq!(outcomes[3].code, Some(0));
15880        assert_eq!(outcomes[3].output_tail, "success");
15881        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15882    }
15883
15884    /// The addendum's second gap: a `verify.gate` command running for real
15885    /// wall-clock time had nothing at all to show for it in `active` before
15886    /// `run_commands` learned to record it — a run could sit in `Gating` for
15887    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
15888    /// what was actually happening. Proven with a genuinely still-running
15889    /// command, not just a before/after check on the final state: a poller
15890    /// task reads the same `run.json` `gate()` is writing, the same way the
15891    /// phone or `magi show` would, while the shell command is still blocked
15892    /// on its own release marker.
15893    #[tokio::test]
15894    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15895        crate::run::pin_test_home();
15896
15897        let tmp = tempfile::tempdir().expect("tempdir");
15898        let repo = tmp.path().join("repo");
15899        std::fs::create_dir_all(&repo).unwrap();
15900        init_repo(&repo);
15901
15902        let mut config = Config::default();
15903        config.verify.gate = vec![
15904            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15905             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15906                .to_owned(),
15907        ];
15908
15909        let mut state = RunState::new(
15910            repo.clone(),
15911            "main".to_owned(),
15912            "deadbeef".to_owned(),
15913            "task".to_owned(),
15914            config,
15915        );
15916        let run_id = state.id.clone();
15917        state.candidates = vec![Candidate {
15918            index: 0,
15919            label: 'A',
15920            agent: "alpha".to_owned(),
15921            branch: "does-not-exist".to_owned(),
15922            worktree: repo.clone(),
15923            summary: String::new(),
15924            stat: String::new(),
15925            files: 0,
15926            commits: 0,
15927            empty: false,
15928            failed: None,
15929            verified_noop: None,
15930            duration_ms: 0,
15931            folded: false,
15932        }];
15933        state.tally = Some(Tally {
15934            first_choice: BTreeMap::from([('A', 1)]),
15935            borda: BTreeMap::new(),
15936            winner: 'A',
15937            rankings: 1,
15938            unanimous_initial: true,
15939            deliberated: false,
15940            changed_votes: 0,
15941            unanimous_final: true,
15942            tie_break: None,
15943            judges: 0,
15944            present: 0,
15945            quorum: 0,
15946            met_quorum: true,
15947            uncontested: Some("only candidate A produced a change".to_owned()),
15948        });
15949        state.reviews = vec![ReviewRound {
15950            round: 1,
15951            head: "deadbeef".to_owned(),
15952            verified_head: None,
15953            verified_at: None,
15954            reviews: Vec::new(),
15955            e2e: Vec::new(),
15956            fix: None,
15957            blocking: 0,
15958            answered: 0,
15959            expected: 0,
15960            clean: true,
15961            verify_retried: false,
15962            e2e_deferred: false,
15963            e2e_defer_reason: None,
15964            progressed: false,
15965            vote_split: false,
15966            reconsideration: Vec::new(),
15967            verdict: None,
15968        }];
15969
15970        let mut runner = Runner {
15971            state,
15972            roles: ResolvedRoles {
15973                implementers: Vec::new(),
15974                judges: Vec::new(),
15975                reviewers: Vec::new(),
15976                fixer: None,
15977                conductor: conductor(),
15978                implementer_roster: Vec::new(),
15979                judge_roster: Vec::new(),
15980                reviewer_roster: Vec::new(),
15981            },
15982            sem: Arc::new(Semaphore::new(1)),
15983            pause: Pause::new(),
15984            interrupt: Pause::new(),
15985        };
15986
15987        let started_marker = repo.join("started.marker");
15988        let release_marker = repo.join("release.marker");
15989        let poller = tokio::spawn(async move {
15990            // Bounded so a regression that never records the task entry
15991            // fails this test in seconds instead of hanging the suite —
15992            // the same shape `a_park_requested_while_a_seat_is_mid_call_
15993            // does_not_cut_it_short` uses for the same reason.
15994            for _ in 0..100 {
15995                if started_marker.exists()
15996                    && let Ok(s) = crate::run::RunState::load(&run_id)
15997                    && let Some(a) = s.active.get("gate")
15998                {
15999                    std::fs::write(&release_marker, b"go").expect("release marker");
16000                    return Some(a.clone());
16001                }
16002                tokio::time::sleep(Duration::from_millis(50)).await;
16003            }
16004            None
16005        });
16006
16007        runner.gate().await.expect("gate");
16008        let captured = poller.await.expect("poller task");
16009        let captured = captured.expect(
16010            "the poller never saw a `gate` task entry in run.json while the command was \
16011             still blocked on its own release marker",
16012        );
16013
16014        assert_eq!(captured.task.as_deref(), Some("gate"));
16015        assert_eq!(captured.node, "gate");
16016        assert_eq!(captured.index, Some(1));
16017        assert_eq!(captured.total, Some(1));
16018        assert!(
16019            captured
16020                .command
16021                .as_deref()
16022                .is_some_and(|c| c.contains("started.marker")),
16023            "{captured:?}"
16024        );
16025
16026        assert!(
16027            runner.state.active.is_empty(),
16028            "the entry must be cleared once the command actually finished: {:?}",
16029            runner.state.active
16030        );
16031        assert!(runner.state.gate_ran);
16032        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16033    }
16034
16035    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16036    /// record for `land`; one with only a Minor, or no reject, leaves none.
16037    #[tokio::test]
16038    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16039        use crate::verdict::{Finding, ReviewVote, Severity};
16040        crate::run::pin_test_home();
16041        let tmp = tempfile::tempdir().expect("tempdir");
16042        let repo = tmp.path().join("repo");
16043        std::fs::create_dir_all(&repo).unwrap();
16044        init_repo(&repo);
16045
16046        for (severity, vote, expect) in [
16047            (Severity::Major, ReviewVote::Reject, true),
16048            (Severity::Minor, ReviewVote::Reject, false),
16049            (Severity::Major, ReviewVote::Approve, false),
16050        ] {
16051            let mut round = review_round(false, 1, 1, 1, false, true);
16052            round.reviews = vec![ReviewRecord {
16053                reviewer: 1,
16054                agent: "alpha".to_owned(),
16055                summary: String::new(),
16056                findings: vec![Finding {
16057                    id: "R1-1-1".to_owned(),
16058                    severity,
16059                    file: None,
16060                    line: None,
16061                    title: "t".to_owned(),
16062                    detail: String::new(),
16063                }],
16064                vote: Some(vote),
16065                failed: None,
16066                duration_ms: 0,
16067                attempts: 0,
16068            }];
16069            let mut state = RunState::new(
16070                repo.clone(),
16071                "main".to_owned(),
16072                "deadbeef".to_owned(),
16073                "task".to_owned(),
16074                Config::default(),
16075            );
16076            state.reviews = vec![round];
16077            let mut runner = Runner {
16078                state,
16079                roles: ResolvedRoles {
16080                    implementers: Vec::new(),
16081                    judges: Vec::new(),
16082                    reviewers: Vec::new(),
16083                    fixer: None,
16084                    conductor: conductor(),
16085                    implementer_roster: Vec::new(),
16086                    judge_roster: Vec::new(),
16087                    reviewer_roster: Vec::new(),
16088                },
16089                sem: Arc::new(Semaphore::new(1)),
16090                pause: Pause::new(),
16091                interrupt: Pause::new(),
16092            };
16093            let shell = runner.state.config.shell();
16094            runner
16095                .stop_reviewing("round budget spent", &shell, &repo)
16096                .await
16097                .expect("stop_reviewing");
16098            assert_eq!(runner.state.status, RunStatus::Gating);
16099            assert_eq!(
16100                runner.state.contested_handoff.is_some(),
16101                expect,
16102                "{severity:?} + {vote:?}"
16103            );
16104        }
16105    }
16106
16107    /// The shape the incident this whole fix responds to actually had: the
16108    /// round budget spent, the last round's own e2e blocked on the shared
16109    /// build cache (held here by a live pid — this test process — exactly
16110    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16111    /// without forking a process). `stop_reviewing` must retry it — not
16112    /// silently leave the round looking untouched (the catch-up-only half of
16113    /// the bug), and not read the contention as a red `e2e` and block the
16114    /// run on it (the other half). Called directly, the same way
16115    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16116    /// above exercises `gate`, so this never needs a real cargo build to
16117    /// reach: the lease is never released, so `with_cache_lease` never gets
16118    /// past acquiring it into anything that would need a real workspace.
16119    #[tokio::test]
16120    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16121        crate::run::pin_test_home();
16122        let home = crate::run::home();
16123
16124        let tmp = tempfile::tempdir().expect("tempdir");
16125        let repo = tmp.path().join("repo");
16126        std::fs::create_dir_all(&repo).unwrap();
16127        init_repo(&repo);
16128        let head = crate::git::rev_parse(&repo, "HEAD")
16129            .await
16130            .expect("rev-parse");
16131        // Unique to this test, so holding its lease cannot collide with
16132        // another test sharing the same process-wide `home`.
16133        let cache_dir = tmp.path().join("target");
16134
16135        let mut config = Config::default();
16136        config.verify.e2e = vec![format!(
16137            "CARGO_TARGET_DIR='{}' test -f README.md",
16138            cache_dir.display()
16139        )];
16140        config.graph.review_rounds = 1;
16141        // Bounded so a regression that does start waiting fails the test in
16142        // seconds, not hangs it.
16143        config.graph.timeout_verify = Some(2);
16144        config.graph.timeout_resource_wait = 2;
16145
16146        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16147        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16148            .expect("no io error acquiring directly")
16149        {
16150            crate::cache::AcquireOutcome::Acquired(g) => g,
16151            crate::cache::AcquireOutcome::Busy(b) => {
16152                panic!("expected the direct acquire to win the lease first: {b:?}")
16153            }
16154        };
16155
16156        let mut state = RunState::new(
16157            repo.clone(),
16158            "main".to_owned(),
16159            head.clone(),
16160            "task".to_owned(),
16161            config,
16162        );
16163        state.candidates = vec![Candidate {
16164            index: 0,
16165            label: 'A',
16166            agent: "alpha".to_owned(),
16167            branch: "does-not-exist".to_owned(),
16168            worktree: repo.clone(),
16169            summary: String::new(),
16170            stat: String::new(),
16171            files: 0,
16172            commits: 0,
16173            empty: false,
16174            failed: None,
16175            verified_noop: None,
16176            duration_ms: 0,
16177            folded: false,
16178        }];
16179        state.tally = Some(Tally {
16180            first_choice: BTreeMap::from([('A', 1)]),
16181            borda: BTreeMap::new(),
16182            winner: 'A',
16183            rankings: 1,
16184            unanimous_initial: true,
16185            deliberated: false,
16186            changed_votes: 0,
16187            unanimous_final: true,
16188            tie_break: None,
16189            judges: 0,
16190            present: 0,
16191            quorum: 0,
16192            met_quorum: true,
16193            uncontested: Some("only candidate A produced a change".to_owned()),
16194        });
16195        // The round budget's last round, deferred: `needs_catchup_run`'s
16196        // other trigger. `stop_reviewing`'s retry machinery must treat this
16197        // exactly like a resource-blocked attempt once it actually runs.
16198        state.reviews = vec![ReviewRound {
16199            round: 1,
16200            head: head.clone(),
16201            verified_head: None,
16202            verified_at: None,
16203            reviews: Vec::new(),
16204            e2e: Vec::new(),
16205            fix: None,
16206            blocking: 1,
16207            answered: 1,
16208            expected: 1,
16209            clean: false,
16210            verify_retried: false,
16211            e2e_deferred: true,
16212            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16213            progressed: false,
16214            vote_split: false,
16215            reconsideration: Vec::new(),
16216            verdict: None,
16217        }];
16218
16219        let mut runner = Runner {
16220            state,
16221            roles: ResolvedRoles {
16222                implementers: Vec::new(),
16223                judges: Vec::new(),
16224                reviewers: Vec::new(),
16225                fixer: None,
16226                conductor: conductor(),
16227                implementer_roster: Vec::new(),
16228                judge_roster: Vec::new(),
16229                reviewer_roster: Vec::new(),
16230            },
16231            sem: Arc::new(Semaphore::new(1)),
16232            pause: Pause::new(),
16233            interrupt: Pause::new(),
16234        };
16235
16236        let shell = runner.state.config.shell();
16237        runner
16238            .stop_reviewing("round budget spent", &shell, &repo)
16239            .await
16240            .expect("stop_reviewing");
16241
16242        let last = runner.state.reviews.last().expect("round record");
16243        assert_eq!(
16244            last.e2e_status(),
16245            E2eStatus::ResourceBlocked,
16246            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16247             failed: {last:?}"
16248        );
16249        assert_eq!(
16250            last.verified_head.as_deref(),
16251            Some(head.as_str()),
16252            "which commit this attempt targeted is known even though nothing finished checking \
16253             it"
16254        );
16255        let first_attempt_at = last
16256            .verified_at
16257            .expect("when this attempt ran is known too");
16258        assert_ne!(
16259            runner.state.status,
16260            RunStatus::Blocked,
16261            "contention is evidence about the machine, not the patch — it must not settle the \
16262             run as blocked: {:?}",
16263            runner.state.status
16264        );
16265        assert!(
16266            !runner
16267                .state
16268                .events
16269                .iter()
16270                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16271            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16272            runner.state.events
16273        );
16274
16275        // The cache is still held: a later reentry must retry the same
16276        // round's verification again — not leave it looking exactly as
16277        // untouched as the first blocked attempt, which is indistinguishable
16278        // from never having tried again at all.
16279        runner
16280            .stop_reviewing("round budget spent", &shell, &repo)
16281            .await
16282            .expect("stop_reviewing retry");
16283        assert_eq!(
16284            runner.state.reviews.len(),
16285            1,
16286            "no new round was started: {:?}",
16287            runner.state.reviews
16288        );
16289        let last = runner.state.reviews.last().expect("round record");
16290        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16291        assert!(
16292            last.verified_at.expect("still known") > first_attempt_at,
16293            "a second reentry must be a fresh attempt, not a stale copy of the first"
16294        );
16295        assert_ne!(runner.state.status, RunStatus::Blocked);
16296
16297        held.release();
16298    }
16299
16300    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
16301    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
16302    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
16303    /// own top-of-function fast path (`review_conclusion`) correctly reads
16304    /// this shape as `None` rather than guessing `Blocked`, and the loop's
16305    /// own `for` range is empty once the round budget is spent, so
16306    /// `review_loop` must retry the check itself rather than silently doing
16307    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
16308    /// above exercises directly, but through `review_loop`'s own entry point
16309    /// this time, proving the wiring between the two rather than just the
16310    /// retry logic in isolation.
16311    #[tokio::test]
16312    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16313        crate::run::pin_test_home();
16314        let home = crate::run::home();
16315
16316        let tmp = tempfile::tempdir().expect("tempdir");
16317        let repo = tmp.path().join("repo");
16318        std::fs::create_dir_all(&repo).unwrap();
16319        init_repo(&repo);
16320        let head = crate::git::rev_parse(&repo, "HEAD")
16321            .await
16322            .expect("rev-parse");
16323        let cache_dir = tmp.path().join("target");
16324
16325        let mut config = Config::default();
16326        config.verify.e2e = vec![format!(
16327            "CARGO_TARGET_DIR='{}' test -f README.md",
16328            cache_dir.display()
16329        )];
16330        config.graph.review_rounds = 1;
16331        config.graph.timeout_verify = Some(2);
16332        config.graph.timeout_resource_wait = 2;
16333
16334        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16335        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16336            .expect("no io error acquiring directly")
16337        {
16338            crate::cache::AcquireOutcome::Acquired(g) => g,
16339            crate::cache::AcquireOutcome::Busy(b) => {
16340                panic!("expected the direct acquire to win the lease first: {b:?}")
16341            }
16342        };
16343
16344        let mut state = RunState::new(
16345            repo.clone(),
16346            "main".to_owned(),
16347            head.clone(),
16348            "task".to_owned(),
16349            config,
16350        );
16351        state.candidates = vec![Candidate {
16352            index: 0,
16353            label: 'A',
16354            agent: "alpha".to_owned(),
16355            branch: "does-not-exist".to_owned(),
16356            worktree: repo.clone(),
16357            summary: String::new(),
16358            stat: String::new(),
16359            files: 0,
16360            commits: 0,
16361            empty: false,
16362            failed: None,
16363            verified_noop: None,
16364            duration_ms: 0,
16365            folded: false,
16366        }];
16367        state.tally = Some(Tally {
16368            first_choice: BTreeMap::from([('A', 1)]),
16369            borda: BTreeMap::new(),
16370            winner: 'A',
16371            rankings: 1,
16372            unanimous_initial: true,
16373            deliberated: false,
16374            changed_votes: 0,
16375            unanimous_final: true,
16376            tie_break: None,
16377            judges: 0,
16378            present: 0,
16379            quorum: 0,
16380            met_quorum: true,
16381            uncontested: Some("only candidate A produced a change".to_owned()),
16382        });
16383        // The exact shape a prior process's `stop_reviewing` would have left
16384        // on disk: the round budget's last round, a real attempt already
16385        // made and already resource-blocked.
16386        state.reviews = vec![ReviewRound {
16387            round: 1,
16388            head: head.clone(),
16389            verified_head: Some(head.clone()),
16390            verified_at: Some(jiff::Timestamp::now()),
16391            reviews: Vec::new(),
16392            e2e: vec![CommandOutcome {
16393                command: format!(
16394                    "CARGO_TARGET_DIR='{}' test -f README.md",
16395                    cache_dir.display()
16396                ),
16397                code: None,
16398                output_tail: "waiting for the shared build cache".to_owned(),
16399                duration_ms: 0,
16400                resource_blocked: true,
16401            }],
16402            fix: None,
16403            blocking: 1,
16404            answered: 1,
16405            expected: 1,
16406            clean: false,
16407            verify_retried: false,
16408            e2e_deferred: false,
16409            e2e_defer_reason: None,
16410            progressed: false,
16411            vote_split: false,
16412            reconsideration: Vec::new(),
16413            verdict: None,
16414        }];
16415
16416        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16417        let mut runner = Runner {
16418            state,
16419            roles: ResolvedRoles {
16420                implementers: Vec::new(),
16421                judges: Vec::new(),
16422                reviewers: Vec::new(),
16423                fixer: None,
16424                conductor: conductor(),
16425                implementer_roster: Vec::new(),
16426                judge_roster: Vec::new(),
16427                reviewer_roster: Vec::new(),
16428            },
16429            sem: Arc::new(Semaphore::new(1)),
16430            pause: Pause::new(),
16431            interrupt: Pause::new(),
16432        };
16433
16434        // The lease is still held throughout, so this reentry's own retry is
16435        // also contended — proving `review_loop` actually tried again (not
16436        // that it happened to succeed) is what the timestamp comparison
16437        // below is for.
16438        runner.review_loop().await.expect("review_loop");
16439
16440        assert_eq!(
16441            runner.state.reviews.len(),
16442            1,
16443            "no new round was started on top of the unresolved one: {:?}",
16444            runner.state.reviews
16445        );
16446        let last = &runner.state.reviews[0];
16447        assert_eq!(
16448            last.e2e_status(),
16449            E2eStatus::ResourceBlocked,
16450            "still contended: {last:?}"
16451        );
16452        assert!(
16453            last.verified_at.expect("still known") > first_attempt_at,
16454            "review_loop must have actually retried the check, not left it exactly as found"
16455        );
16456        assert_ne!(
16457            runner.state.status,
16458            RunStatus::Blocked,
16459            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16460            runner.state.status
16461        );
16462
16463        held.release();
16464    }
16465
16466    #[tokio::test]
16467    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16468        crate::run::pin_test_home();
16469        for status in [RunStatus::Blocked, RunStatus::Gating] {
16470            let mut runner = runner_at(status);
16471            runner.state.config.graph.land = true;
16472            runner.state.gate_ran = true;
16473            runner.state.merge = Some(MergeOutcome {
16474                mode: MergeMode::Pr,
16475                ok: false,
16476                detail: "the fixer produced no commit".into(),
16477                empty: false,
16478            });
16479            let url = "https://example.invalid/o/r/pull/1";
16480            runner.state.pr = Some(crate::run::PrRecord {
16481                url: url.into(),
16482                number: 1,
16483                state: "open".into(),
16484                checks: "red".into(),
16485                round: 1,
16486                rounds: 1,
16487                red_at_merge: Vec::new(),
16488            });
16489            runner.state.save().unwrap();
16490            // The nonexistent repository makes observation fail before a forge
16491            // command can run. We still must reach land, rather than silently
16492            // return from the graph with its old gate and merge records.
16493            runner.execute_graph().await.unwrap();
16494            assert_eq!(runner.state.status, RunStatus::Blocked);
16495            assert!(runner.state.land_sync_pending);
16496            assert!(runner.state.events.iter().any(|event|
16497                event.node == "land" && event.message == format!("watching {url}")));
16498            assert_eq!(
16499                runner.state.merge.as_ref().unwrap().detail,
16500                "the fixer produced no commit"
16501            );
16502        }
16503    }
16504
16505    #[tokio::test]
16506    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16507        crate::run::pin_test_home();
16508        let tmp = tempfile::tempdir().expect("tempdir");
16509        let repo = tmp.path().join("repo");
16510        std::fs::create_dir_all(&repo).unwrap();
16511        init_repo(&repo);
16512
16513        let mut config = Config::default();
16514        config.merge.mode = MergeMode::Pr;
16515        config.graph.land = true;
16516        config.graph.land_approval = false;
16517
16518        let mut state = RunState::new(
16519            repo.clone(),
16520            "main".to_owned(),
16521            "deadbeef".to_owned(),
16522            "task".to_owned(),
16523            config,
16524        );
16525        state.candidates = vec![Candidate {
16526            index: 0,
16527            label: 'A',
16528            agent: "alpha".to_owned(),
16529            branch: "does-not-exist".to_owned(),
16530            worktree: repo.clone(),
16531            summary: String::new(),
16532            stat: String::new(),
16533            files: 0,
16534            commits: 0,
16535            empty: false,
16536            failed: None,
16537            verified_noop: None,
16538            duration_ms: 0,
16539            folded: false,
16540        }];
16541        state.tally = Some(Tally {
16542            first_choice: BTreeMap::from([('A', 1)]),
16543            borda: BTreeMap::new(),
16544            winner: 'A',
16545            rankings: 1,
16546            unanimous_initial: true,
16547            deliberated: false,
16548            changed_votes: 0,
16549            unanimous_final: true,
16550            tie_break: None,
16551            judges: 0,
16552            present: 0,
16553            quorum: 0,
16554            met_quorum: true,
16555            uncontested: Some("only candidate A produced a change".to_owned()),
16556        });
16557        state.reviews = vec![ReviewRound {
16558            round: 1,
16559            head: "deadbeef".to_owned(),
16560            verified_head: None,
16561            verified_at: None,
16562            reviews: Vec::new(),
16563            e2e: Vec::new(),
16564            fix: None,
16565            blocking: 0,
16566            answered: 0,
16567            expected: 0,
16568            clean: true,
16569            verify_retried: false,
16570            e2e_deferred: false,
16571            e2e_defer_reason: None,
16572            progressed: false,
16573            vote_split: false,
16574            reconsideration: Vec::new(),
16575            verdict: None,
16576        }];
16577        state.gate = vec![CommandOutcome {
16578            command: "test".to_owned(),
16579            code: Some(0),
16580            output_tail: String::new(),
16581            duration_ms: 0,
16582            resource_blocked: false,
16583        }];
16584        state.gate_ran = true;
16585        // A first pass through `merge` already pushed and opened this pull
16586        // request; `status` is `Landing` because a previous call into `land`
16587        // parked or was interrupted before it reached a terminal outcome.
16588        state.status = RunStatus::Landing;
16589        state.merge = Some(MergeOutcome {
16590            mode: MergeMode::Pr,
16591            ok: true,
16592            detail: "https://example.invalid/x/y/pull/1".to_owned(),
16593            empty: false,
16594        });
16595
16596        // The Landing-resume shortcut calls `run_land` directly rather than
16597        // through `merge`, which is exactly the call site that used to skip
16598        // `settle_questions` - see the fixture below.
16599        ask_test_home();
16600        // A real resume loads a persisted run; driver ownership is checked
16601        // against that record before the Landing shortcut is entered.
16602        state.save().expect("save landing run");
16603        let store = ask::Questions::open();
16604        let q = ask_open_question(&store, &state.id);
16605
16606        let mut runner = Runner {
16607            state,
16608            roles: ResolvedRoles {
16609                implementers: Vec::new(),
16610                judges: Vec::new(),
16611                reviewers: Vec::new(),
16612                fixer: None,
16613                conductor: conductor(),
16614                implementer_roster: Vec::new(),
16615                judge_roster: Vec::new(),
16616                reviewer_roster: Vec::new(),
16617            },
16618            sem: Arc::new(Semaphore::new(1)),
16619            pause: Pause::new(),
16620            interrupt: Pause::new(),
16621        };
16622
16623        // `execute`, not `merge` directly: the Landing-resume shortcut lives
16624        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
16625        // exactly because `review_loop` would otherwise clobber the marker
16626        // first.
16627        runner.execute().await.expect("execute");
16628
16629        assert_eq!(
16630            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16631            Some("https://example.invalid/x/y/pull/1"),
16632            "reentry must not push again or open a second pull request over the \
16633             one `land` is already watching"
16634        );
16635        assert_ne!(
16636            runner.state.status,
16637            RunStatus::Landing,
16638            "land could not actually reach the fake pull request, so it must \
16639             have given up rather than left the run silently parked forever"
16640        );
16641        // `land` could not reach the fake pull request, so it gave up into
16642        // `Blocked` - still resumable, so the question must not have been
16643        // swept just because this branch now also calls `settle_questions`.
16644        assert_eq!(runner.state.status, RunStatus::Blocked);
16645        assert!(
16646            store.get(&q.id).unwrap().status.open(),
16647            "Blocked is still alive; settle_questions must have been a no-op here"
16648        );
16649    }
16650
16651    fn state_with_round(round: ReviewRound) -> RunState {
16652        let mut s = RunState::new(
16653            PathBuf::from("/repo"),
16654            "main".to_owned(),
16655            "abc1234".to_owned(),
16656            "add retries".to_owned(),
16657            Config::default(),
16658        );
16659        s.reviews = vec![round];
16660        s
16661    }
16662
16663    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16664        crate::verdict::Finding {
16665            id: id.to_owned(),
16666            severity,
16667            file: None,
16668            line: None,
16669            title: title.to_owned(),
16670            detail: String::new(),
16671        }
16672    }
16673
16674    #[test]
16675    fn pr_body_names_open_findings_and_declined_ones() {
16676        let round = ReviewRound {
16677            round: 2,
16678            head: "deadbee".to_owned(),
16679            verified_head: None,
16680            verified_at: None,
16681            reviews: vec![ReviewRecord {
16682                attempts: 0,
16683                reviewer: 1,
16684                agent: "alpha".to_owned(),
16685                summary: String::new(),
16686                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16687                vote: None,
16688                failed: None,
16689                duration_ms: 0,
16690            }],
16691            e2e: vec![CommandOutcome {
16692                command: "cargo test".to_owned(),
16693                code: Some(0),
16694                output_tail: String::new(),
16695                duration_ms: 0,
16696                resource_blocked: false,
16697            }],
16698            verify_retried: false,
16699            e2e_deferred: false,
16700            e2e_defer_reason: None,
16701            fix: Some(FixRecord {
16702                agent: "alpha".to_owned(),
16703                addressed: Vec::new(),
16704                rejected: vec![crate::verdict::Rejection {
16705                    id: "R1-1-1".to_owned(),
16706                    why: "not reachable from any caller".to_owned(),
16707                }],
16708                notes: String::new(),
16709                committed: true,
16710                failed: None,
16711                duration_ms: 0,
16712                continuation: None,
16713            }),
16714            blocking: 0,
16715            answered: 1,
16716            expected: 1,
16717            clean: false,
16718            progressed: true,
16719            vote_split: false,
16720            reconsideration: Vec::new(),
16721            verdict: None,
16722        };
16723        let state = state_with_round(round);
16724        let body = pr_message(&state, 'A').body;
16725
16726        assert!(body.contains("add retries"), "the task must still be there");
16727        assert!(body.contains("R2-1-1"), "{body}");
16728        assert!(!body.contains("unused import"), "{body}");
16729        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16730        assert!(
16731            !body.contains("not reachable from any caller"),
16732            "the reason it was declined: {body}"
16733        );
16734    }
16735
16736    #[test]
16737    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16738        let round = ReviewRound {
16739            round: 1,
16740            head: "deadbee".to_owned(),
16741            verified_head: None,
16742            verified_at: None,
16743            reviews: vec![ReviewRecord {
16744                attempts: 0,
16745                reviewer: 1,
16746                agent: "alpha".to_owned(),
16747                summary: String::new(),
16748                findings: Vec::new(),
16749                vote: None,
16750                failed: None,
16751                duration_ms: 0,
16752            }],
16753            e2e: Vec::new(),
16754            verify_retried: false,
16755            e2e_deferred: false,
16756            e2e_defer_reason: None,
16757            fix: None,
16758            blocking: 0,
16759            answered: 1,
16760            expected: 1,
16761            clean: true,
16762            progressed: false,
16763            vote_split: false,
16764            reconsideration: Vec::new(),
16765            verdict: None,
16766        };
16767        let state = state_with_round(round);
16768        let body = pr_message(&state, 'A').body;
16769        assert!(!body.contains("Open review findings"), "{body}");
16770        assert!(!body.contains("Declined"), "{body}");
16771    }
16772
16773    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16774        let mut state = RunState::new(
16775            PathBuf::from("/repo"),
16776            "main".to_owned(),
16777            "abc1234".to_owned(),
16778            instruction.to_owned(),
16779            Config::default(),
16780        );
16781        state.candidates.push(Candidate {
16782            index: 0,
16783            label: 'A',
16784            agent: "alpha".to_owned(),
16785            branch: "magi/x/A".to_owned(),
16786            worktree: PathBuf::from("/wt"),
16787            summary: summary.to_owned(),
16788            stat: String::new(),
16789            files: 1,
16790            commits: 1,
16791            empty: false,
16792            failed: None,
16793            verified_noop: None,
16794            folded: false,
16795            duration_ms: 0,
16796        });
16797        state
16798    }
16799
16800    fn review_state(subjects: &[&str]) -> RunState {
16801        let mut state = state_with_summary(
16802            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16803            "",
16804        );
16805        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16806        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16807        state
16808    }
16809
16810    /// A branch rebased onto a main that moved past the recorded
16811    /// `base_commit` is titled from its own first commit, never main's.
16812    #[tokio::test]
16813    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16814        ask_test_home();
16815        let tmp = tempfile::tempdir().unwrap();
16816        let repo = tmp.path().join("repo");
16817        std::fs::create_dir_all(&repo).unwrap();
16818        init_repo(&repo);
16819        let origin = tmp.path().join("origin.git");
16820        let g = |dir: &Path, args: &[&str]| {
16821            let out = std::process::Command::new("git")
16822                .args(args)
16823                .current_dir(dir)
16824                .quiet()
16825                .output()
16826                .expect("spawn git");
16827            assert!(
16828                out.status.success(),
16829                "git {args:?}: {}",
16830                String::from_utf8_lossy(&out.stderr)
16831            );
16832        };
16833        g(
16834            tmp.path(),
16835            &[
16836                "clone",
16837                "--bare",
16838                "-q",
16839                repo.to_str().unwrap(),
16840                origin.to_str().unwrap(),
16841            ],
16842        );
16843        g(
16844            &repo,
16845            &["remote", "add", "origin", origin.to_str().unwrap()],
16846        );
16847        let c1 = git::rev_parse(&repo, "main").await.unwrap();
16848
16849        // Main moves on; the branch is built on top of the new main.
16850        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16851        g(&repo, &["add", "-A"]);
16852        g(
16853            &repo,
16854            &["commit", "-q", "-m", "chore(deps): update a crate"],
16855        );
16856        g(&repo, &["push", "-q", "origin", "main"]);
16857        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16858        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16859        g(&repo, &["add", "-A"]);
16860        g(
16861            &repo,
16862            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16863        );
16864        g(&repo, &["checkout", "-q", "main"]);
16865
16866        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16867        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16868        // Without a readable tracking ref the recorded base's merge base is used.
16869        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16870        assert_eq!(fallback, c1);
16871
16872        let mut state = review_state(&[
16873            "chore(deps): update a crate",
16874            "fix(daemon): apply a chosen action",
16875        ]);
16876        state.repo = repo.clone();
16877        state.base_branch = "main".to_owned();
16878        state.base_commit = c1;
16879        refresh_reviewed_commits(&mut state, "feat/own").await;
16880        assert_eq!(
16881            state.reviewed_commits,
16882            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16883        );
16884        assert_eq!(
16885            review_title(&state).as_deref(),
16886            Some("fix(daemon): apply a chosen action")
16887        );
16888        assert_eq!(
16889            leaked_subjects(&state, "feat/own").await,
16890            Some(vec!["chore(deps): update a crate".to_owned()])
16891        );
16892        // A stale tracking ref is still used when the fetch fails, but the
16893        // leak list is withheld.
16894        state.config.merge.remote = "nowhere".to_owned();
16895        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16896    }
16897
16898    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
16899    /// with `origin/main` one commit behind it, and a summarizing agent that
16900    /// prints whatever `reply` holds.
16901    struct SummaryFixture {
16902        _tmp: tempfile::TempDir,
16903        repo: PathBuf,
16904        reply: PathBuf,
16905        runner: Runner,
16906    }
16907
16908    fn summary_fixture() -> SummaryFixture {
16909        ask_test_home();
16910        let tmp = tempfile::tempdir().unwrap();
16911        let repo = tmp.path().join("repo");
16912        std::fs::create_dir_all(&repo).unwrap();
16913        init_repo(&repo);
16914        let origin = tmp.path().join("origin.git");
16915        let g = |args: &[&str]| {
16916            let out = std::process::Command::new("git")
16917                .args(args)
16918                .current_dir(&repo)
16919                .quiet()
16920                .output()
16921                .expect("spawn git");
16922            assert!(
16923                out.status.success(),
16924                "git {args:?}: {}",
16925                String::from_utf8_lossy(&out.stderr)
16926            );
16927        };
16928        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16929        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16930        g(&["push", "-q", "origin", "main"]);
16931        g(&["checkout", "-q", "-b", "feat/own"]);
16932        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16933        g(&["add", "-A"]);
16934        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16935        let reply = tmp.path().join("reply.txt");
16936        std::fs::write(
16937            &reply,
16938            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16939        )
16940        .unwrap();
16941        let mut runner = runner_at(RunStatus::Gating);
16942        let mut state = review_state(&["feat(own): add the own file"]);
16943        state.repo = repo.clone();
16944        state.base_branch = "main".to_owned();
16945        state.candidates[0].branch = "feat/own".to_owned();
16946        state.candidates[0].worktree = repo.clone();
16947        let mut author = spec("alpha");
16948        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16949        author
16950            .env
16951            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16952        state.config.agents = vec![author];
16953        runner.state = state;
16954        SummaryFixture {
16955            _tmp: tmp,
16956            repo,
16957            reply,
16958            runner,
16959        }
16960    }
16961
16962    async fn posted(f: &mut SummaryFixture) -> PrMessage {
16963        let winner = f.runner.state.candidates[0].clone();
16964        f.runner
16965            .guarded_pr_message(&winner, None, true)
16966            .await
16967            .unwrap()
16968            .expect("an English description never parks")
16969    }
16970
16971    fn clean_round_with_minor_findings() -> ReviewRound {
16972        let mut round = passing_round(2, None);
16973        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16974        round.reviews = vec![ReviewRecord {
16975            attempts: 0,
16976            reviewer: 1,
16977            agent: "alpha".into(),
16978            summary: "内部レビュー".into(),
16979            findings: vec![
16980                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16981                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16982            ],
16983            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16984            failed: None,
16985            duration_ms: 0,
16986        }];
16987        round
16988    }
16989
16990    #[tokio::test]
16991    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16992        for review_only in [false, true] {
16993            let mut f = summary_fixture();
16994            if !review_only {
16995                f.runner.state.candidates[0].agent = "alpha".into();
16996            }
16997            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16998            assert!(f.runner.state.reviews[0].clean);
16999            assert!(f.runner.state.reviews[0].fix.is_none());
17000            assert!(f.runner.state.open_findings().is_empty());
17001            assert_eq!(
17002                crate::pr_outcome::required_finding_ids(&f.runner.state),
17003                ["R2-1-1", "R2-1-2"]
17004            );
17005            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17006            let evidence = crate::pr_outcome::summary_evidence(
17007                &f.runner.state,
17008                crate::pr_outcome::FinalHead::At(&head),
17009            );
17010            assert!(
17011                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17012            );
17013            // The ordinary reply omits both IDs: no cached description is adopted.
17014            let omitted = posted(&mut f).await;
17015            assert!(omitted.body.contains("No description"));
17016            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17017            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17018            assert!(f.runner.state.change_summary.is_none());
17019            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17020            let described = posted(&mut f).await;
17021            assert!(described.body.contains("improve documentation for callers"));
17022            assert!(described.body.contains("remaining edge case"));
17023            assert!(described.body.contains("## Open review findings"));
17024            assert!(!described.body.contains("未修正"));
17025        }
17026    }
17027
17028    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17029        let mut f = summary_fixture();
17030        if !review_only {
17031            f.runner.state.candidates[0].agent = "alpha".into();
17032        }
17033        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17034        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17035        fixed
17036            .fix
17037            .as_mut()
17038            .unwrap()
17039            .rejected
17040            .push(crate::verdict::Rejection {
17041                id: "R1-1-1".into(),
17042                why: "呼び出されないコードのため辞退".into(),
17043            });
17044        let mut clean = clean_round_with_minor_findings();
17045        clean.verified_head = Some(head.clone());
17046        f.runner.state.reviews = vec![fixed, clean];
17047        ok_gate(&mut f.runner.state, Some(&head));
17048        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17049        f.runner.state.config.agents.push(scripted("beta", reply));
17050        f.runner.state.config.roles.language_judge =
17051            Some(crate::config::AgentChoice::One("beta".into()));
17052        f
17053    }
17054
17055    #[tokio::test]
17056    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17057        for review_only in [false, true] {
17058            let mut f = rewrite_fixture_with_findings(review_only,
17059                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17060            let m = posted(&mut f).await;
17061            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17062            let expected =
17063                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17064            assert!(m.body.contains(&expected));
17065            assert!(m.body.contains("## Review rounds"));
17066            assert!(
17067                m.body
17068                    .contains("Gate (`verify.gate`): passed on the final commit")
17069            );
17070            assert!(
17071                m.body
17072                    .contains("Tests (`verify.e2e`): passed on the final commit")
17073            );
17074            let c = f.runner.state.change_summary.as_ref().unwrap();
17075            assert!(
17076                m.body
17077                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17078            );
17079            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17080            assert!(crate::pr_outcome::covers_required_findings(
17081                &f.runner.state,
17082                &m.body
17083            ));
17084            assert!(m.body.contains("documentation remains incomplete"));
17085            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17086            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17087            assert!(f.runner.state.github_text.is_none());
17088        }
17089    }
17090
17091    #[tokio::test]
17092    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17093        for review_only in [false, true] {
17094            for reply in [
17095                "TITLE: fix: public description\nA short English description.",
17096                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17097            ] {
17098                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17099                let winner = f.runner.state.candidates[0].clone();
17100                let posted = f
17101                    .runner
17102                    .guarded_pr_message(&winner, None, true)
17103                    .await
17104                    .unwrap();
17105                assert!(posted.is_none());
17106                assert!(f.runner.state.parked);
17107                let withheld = f.runner.state.github_text.as_ref().unwrap();
17108                assert!(
17109                    withheld.rewrite.is_none(),
17110                    "an incomplete rewrite must not be selectable"
17111                );
17112                let q = ask::Questions::open()
17113                    .get(withheld.question.as_ref().unwrap())
17114                    .unwrap();
17115                assert!(
17116                    !q.choices
17117                        .contains(&crate::github_text::USE_REWRITE.to_owned())
17118                );
17119                assert!(
17120                    q.detail.contains("R1-1-1")
17121                        && q.detail.contains("R2-1-1")
17122                        && q.detail.contains("R2-1-2")
17123                );
17124            }
17125        }
17126    }
17127
17128    #[tokio::test]
17129    async fn japanese_reports_stay_local_in_both_public_run_modes() {
17130        for review_only in [false, true] {
17131            let mut f = summary_fixture();
17132            if !review_only {
17133                f.runner.state.candidates[0].agent = "alpha".into();
17134            }
17135            f.runner.state.instruction = "内部タスクの説明".into();
17136            f.runner.state.candidates[0].summary = "内部の候補報告".into();
17137            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17138            f.runner.state.reviews = vec![
17139                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17140                round_with_notes(2, Some("二回目の内部修正報告です。")),
17141                passing_round(3, Some(&head)),
17142            ];
17143            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17144            ok_gate(&mut f.runner.state, Some(&head));
17145            let captured = f.repo.join("public-prompt.txt");
17146            let author = &mut f.runner.state.config.agents[0];
17147            author.command = vec![
17148                "sh".into(),
17149                "-c".into(),
17150                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17151            ];
17152            author.env.insert(
17153                "PROMPT_FILE".into(),
17154                captured.to_string_lossy().into_owned(),
17155            );
17156            let m = posted(&mut f).await;
17157            assert!(m.body.contains("Adds `own.txt`"));
17158            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17159            assert!(m.body.contains("R1-1-1"));
17160            assert!(
17161                m.body
17162                    .contains("Gate (`verify.gate`): passed on the final commit")
17163            );
17164            assert!(
17165                m.body
17166                    .contains("Tests (`verify.e2e`): passed on the final commit")
17167            );
17168            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17169            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17170            let prompt = std::fs::read_to_string(captured).unwrap();
17171            assert!(prompt.contains(&format!(
17172                "git diff {} {head}",
17173                f.runner.state.change_summary.as_ref().unwrap().base
17174            )));
17175            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17176            assert_eq!(
17177                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17178                "二回目の内部修正報告です。"
17179            );
17180        }
17181    }
17182
17183    #[tokio::test]
17184    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17185        for review_only in [false, true] {
17186            let mut f = summary_fixture();
17187            if !review_only {
17188                f.runner.state.candidates[0].agent = "alpha".into();
17189            }
17190            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17191            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17192            std::fs::write(&f.reply, "no usable description").unwrap();
17193            let failed = posted(&mut f).await;
17194            assert!(
17195                failed
17196                    .body
17197                    .contains("No description of what this change does could be generated")
17198            );
17199            assert!(failed.body.is_ascii());
17200            assert!(f.runner.state.github_text.is_none());
17201            assert!(f.runner.state.change_summary.is_none());
17202            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17203            let evidence = crate::pr_outcome::summary_evidence(
17204                &f.runner.state,
17205                crate::pr_outcome::FinalHead::At(&head),
17206            );
17207            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17208            assert!(
17209                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17210                    .exists()
17211            );
17212            f.runner.state =
17213                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17214            std::fs::write(
17215                &f.reply,
17216                "TITLE: feat(own): add the own file\nAdds the committed own file.",
17217            )
17218            .unwrap();
17219            let retried = posted(&mut f).await;
17220            assert!(retried.body.contains("Adds the committed own file."));
17221            assert_eq!(
17222                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17223                "日本語の修正報告"
17224            );
17225        }
17226    }
17227
17228    #[tokio::test]
17229    async fn same_head_verification_and_findings_invalidate_public_prose() {
17230        let mut f = summary_fixture();
17231        posted(&mut f).await;
17232        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17233        let original = f
17234            .runner
17235            .state
17236            .change_summary
17237            .as_ref()
17238            .unwrap()
17239            .evidence
17240            .clone();
17241        ok_gate(&mut f.runner.state, Some(&head));
17242        std::fs::write(
17243            &f.reply,
17244            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17245        )
17246        .unwrap();
17247        let verified = posted(&mut f).await;
17248        assert!(verified.body.contains("now verified"));
17249        assert_ne!(
17250            f.runner.state.change_summary.as_ref().unwrap().evidence,
17251            original
17252        );
17253        let mut round = round_with_notes(1, Some("内部報告"));
17254        round.reviews.push(ReviewRecord {
17255            attempts: 0,
17256            reviewer: 1,
17257            agent: "alpha".into(),
17258            summary: "内部レビュー".into(),
17259            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17260            vote: None,
17261            failed: None,
17262            duration_ms: 0,
17263        });
17264        round
17265            .fix
17266            .as_mut()
17267            .unwrap()
17268            .rejected
17269            .push(crate::verdict::Rejection {
17270                id: "R1-1-1".into(),
17271                why: "呼び出し元から到達できない".into(),
17272            });
17273        f.runner.state.reviews = vec![round];
17274        // A description omitting the required ID is unusable, even with a title.
17275        let omitted = posted(&mut f).await;
17276        assert!(omitted.body.contains("No description"));
17277        assert!(omitted.body.contains("R1-1-1"));
17278        assert!(!omitted.body.contains("到達"));
17279        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17280        let described = posted(&mut f).await;
17281        assert!(described.body.contains("unreachable from callers"));
17282        assert!(described.body.contains("R1-1-1"));
17283        assert!(!described.body.contains("到達"));
17284        let evidence = f
17285            .runner
17286            .state
17287            .change_summary
17288            .as_ref()
17289            .unwrap()
17290            .evidence
17291            .clone();
17292        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17293        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17294        assert!(posted(&mut f).await.body.contains("rationale updated"));
17295        assert_ne!(
17296            f.runner.state.change_summary.as_ref().unwrap().evidence,
17297            evidence
17298        );
17299        // Old serialized summaries have no evidence fingerprint and must regenerate.
17300        f.runner
17301            .state
17302            .change_summary
17303            .as_mut()
17304            .unwrap()
17305            .evidence
17306            .clear();
17307        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17308        assert!(posted(&mut f).await.body.contains("current specification"));
17309    }
17310
17311    #[tokio::test]
17312    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17313        let mut f = summary_fixture();
17314        let m = posted(&mut f).await;
17315        assert_eq!(m.title, "feat(own): add the own file");
17316        assert!(
17317            m.body
17318                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17319            "{}",
17320            m.body
17321        );
17322        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17323        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17324        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17325        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17326        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17327        let kept = f.runner.state.change_summary.as_ref().unwrap();
17328        assert_eq!(
17329            (kept.head.as_str(), kept.base.as_str()),
17330            (head.as_str(), base.as_str())
17331        );
17332        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17333    }
17334
17335    #[tokio::test]
17336    async fn review_only_summary_receives_the_configured_github_language() {
17337        let mut f = summary_fixture();
17338        f.runner.state.config.graph.github_language = "ja".into();
17339        let captured = f.repo.join("summary-prompt.txt");
17340        let author = &mut f.runner.state.config.agents[0];
17341        author.command = vec![
17342            "sh".into(),
17343            "-c".into(),
17344            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17345        ];
17346        author.env.insert(
17347            "PROMPT_FILE".into(),
17348            captured.to_string_lossy().into_owned(),
17349        );
17350        std::fs::write(
17351            &f.reply,
17352            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17353        )
17354        .unwrap();
17355        let m = posted(&mut f).await;
17356        let prompt = std::fs::read_to_string(captured).unwrap();
17357        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17358        assert!(!prompt.contains("in English"));
17359        assert_eq!(m.title, "feat(own): ファイルを追加");
17360        assert!(m.body.contains("own.txt を追加します。"));
17361    }
17362
17363    #[tokio::test]
17364    async fn review_fixes_come_after_the_whole_change_summary() {
17365        let mut f = summary_fixture();
17366        f.runner.state.reviews = vec![
17367            round_with_notes(1, Some("handled the PATH case")),
17368            round_with_notes(2, None),
17369        ];
17370        let m = posted(&mut f).await;
17371        let summary = m.body.find("## Summary").expect("summary");
17372        let fixes = m.body.find("## Review rounds").expect("review rounds");
17373        assert!(summary < fixes, "{}", m.body);
17374        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17375        assert!(m.body.contains("## Final verification"), "{}", m.body);
17376    }
17377
17378    #[tokio::test]
17379    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17380        let mut f = summary_fixture();
17381        let first = posted(&mut f).await;
17382        let first_head = f.runner.state.change_summary.clone().unwrap().head;
17383        // The same head reuses the record: a different reply is not read.
17384        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17385        let again = posted(&mut f).await;
17386        assert_eq!((again.title, again.body), (first.title, first.body));
17387
17388        let g = |args: &[&str]| {
17389            let out = std::process::Command::new("git")
17390                .args(args)
17391                .current_dir(&f.repo)
17392                .quiet()
17393                .output()
17394                .expect("spawn git");
17395            assert!(out.status.success(), "git {args:?}");
17396        };
17397        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17398        g(&["add", "-A"]);
17399        g(&["commit", "-q", "-m", "fix(own): handle more"]);
17400        let second = posted(&mut f).await;
17401        let kept = f.runner.state.change_summary.clone().unwrap();
17402        assert_ne!(kept.head, first_head);
17403        assert!(second.body.contains("Not read yet."), "{}", second.body);
17404        assert!(!second.body.contains("so the first summary is read"));
17405        assert_eq!(second.title, "feat(own): other");
17406    }
17407
17408    #[tokio::test]
17409    async fn an_unavailable_summary_is_stated_and_never_invented() {
17410        let mut f = summary_fixture();
17411        std::fs::write(&f.reply, "no title line here").unwrap();
17412        let m = posted(&mut f).await;
17413        assert!(
17414            m.body
17415                .contains("No description of what this change does could be generated"),
17416            "{}",
17417            m.body
17418        );
17419        assert!(m.body.contains("internal reports remain in local records"));
17420        assert!(f.runner.state.change_summary.is_none());
17421        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17422        // A preview asks nobody and claims nothing.
17423        let winner = f.runner.state.candidates[0].clone();
17424        let preview = f
17425            .runner
17426            .guarded_pr_message(&winner, None, false)
17427            .await
17428            .unwrap()
17429            .unwrap();
17430        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17431    }
17432
17433    #[test]
17434    fn pr_message_review_single_commit_uses_its_subject() {
17435        let state = review_state(&["feat(nats): per-role user"]);
17436        let m = pr_message(&state, 'A');
17437        assert_eq!(m.title, "feat(nats): per-role user");
17438        assert!(!m.body.contains("Review the work already"), "{}", m.body);
17439        assert!(m.body.contains("## Commits under review"), "{}", m.body);
17440    }
17441
17442    #[test]
17443    fn pr_message_review_multi_commit_takes_the_oldest() {
17444        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17445        let m = pr_message(&state, 'A');
17446        assert_eq!(m.title, "feat: the change");
17447        for s in ["feat: the change", "fix: typo", "fix: again"] {
17448            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17449        }
17450    }
17451
17452    #[test]
17453    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17454        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17455            let mut state = review_state(&[first, "fix: later fixup"]);
17456            state.candidates[0].branch = "機能/ログイン".to_owned();
17457            let m = pr_message(&state, 'A');
17458            assert!(
17459                m.title.starts_with("chore: land candidate A of run"),
17460                "{}",
17461                m.title
17462            );
17463        }
17464    }
17465
17466    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17467        BranchFacts {
17468            commits: commits
17469                .iter()
17470                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17471                .collect(),
17472            stat: stat.to_owned(),
17473        }
17474    }
17475
17476    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17477        let mut r = review_round(true, 0, 1, 1, true, true);
17478        r.round = round;
17479        r.fix = notes.map(|n| FixRecord {
17480            agent: "fixer".to_owned(),
17481            addressed: Vec::new(),
17482            rejected: Vec::new(),
17483            notes: n.to_owned(),
17484            committed: true,
17485            failed: None,
17486            duration_ms: 0,
17487            continuation: None,
17488        });
17489        r
17490    }
17491
17492    #[test]
17493    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17494        let state = review_state(&["ignored"]);
17495        let f = facts(
17496            &[
17497                (
17498                    "fix(login): resolve PATH on macOS",
17499                    "Login shells skip rc files.",
17500                ),
17501                ("fix: address review", ""),
17502            ],
17503            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17504        );
17505        let m = pr_message_with(&state, 'A', Some(&f));
17506        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17507        assert!(
17508            m.body
17509                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
17510            "{}",
17511            m.body
17512        );
17513        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17514        assert!(m.body.contains("## Diff stat"), "{}", m.body);
17515        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17516        for banned in [
17517            "Review the work already",
17518            "no task statement",
17519            "Original task",
17520        ] {
17521            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17522        }
17523        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17524    }
17525
17526    #[test]
17527    fn pr_message_review_truncates_a_huge_first_commit_body() {
17528        let state = review_state(&["ignored"]);
17529        let f = facts(
17530            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17531            "s",
17532        );
17533        let m = pr_message_with(&state, 'A', Some(&f));
17534        assert!(m.body.len() < 30_000, "{}", m.body.len());
17535        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17536        assert!(
17537            m.body.contains("- ... 1 more commit(s)"),
17538            "{}",
17539            m.body.len()
17540        );
17541        assert!(m.body.ends_with("magi:candidate-a\n"));
17542    }
17543
17544    #[test]
17545    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17546        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17547        assert_eq!(m.title, "feat: x");
17548        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17549        assert!(!m.body.contains("Diff stat"), "{}", m.body);
17550        assert!(!m.body.contains("no task statement"), "{}", m.body);
17551    }
17552
17553    #[test]
17554    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17555        let mut state = review_state(&["日本語の件名"]);
17556        state.candidates[0].branch = "fix/macos-login-path".to_owned();
17557        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17558        state.candidates[0].branch = "機能/ログイン".to_owned();
17559        assert!(
17560            pr_message(&state, 'A')
17561                .title
17562                .starts_with("chore: land candidate A")
17563        );
17564    }
17565
17566    #[test]
17567    fn pr_message_review_fixes_survive_a_clean_final_round() {
17568        let mut state = review_state(&["feat: x"]);
17569        state.reviews = vec![
17570            round_with_notes(1, Some("handled the PATH case")),
17571            round_with_notes(2, None),
17572        ];
17573        let body = pr_message(&state, 'A').body;
17574        assert!(
17575            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17576            "{body}"
17577        );
17578        assert!(body.contains("### Round 1"), "{body}");
17579
17580        state.reviews = vec![
17581            round_with_notes(1, Some("first fix")),
17582            round_with_notes(2, Some("")),
17583            round_with_notes(3, Some("second fix")),
17584            round_with_notes(4, None),
17585        ];
17586        let body = pr_message(&state, 'A').body;
17587        assert!(
17588            body.contains("### Round 1") && !body.contains("first fix"),
17589            "{body}"
17590        );
17591        assert!(
17592            body.contains("### Round 3") && !body.contains("second fix"),
17593            "{body}"
17594        );
17595        assert!(body.contains("### Round 2"), "{body}");
17596    }
17597
17598    const FINAL: &str = "abcdef1234567890";
17599
17600    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17601        let mut r = round_with_notes(round, None);
17602        r.verified_head = verified.map(str::to_owned);
17603        r
17604    }
17605
17606    fn ok_gate(state: &mut RunState, head: Option<&str>) {
17607        state.gate = vec![CommandOutcome {
17608            command: "gate".to_owned(),
17609            code: Some(0),
17610            output_tail: String::new(),
17611            duration_ms: 0,
17612            resource_blocked: false,
17613        }];
17614        state.gate_ran = true;
17615        state.gate_head = head.map(str::to_owned);
17616    }
17617
17618    fn body_at(state: &RunState) -> String {
17619        pr_message_raw(
17620            state,
17621            'A',
17622            None,
17623            crate::pr_outcome::FinalHead::At(FINAL),
17624            ChangeNote::NotAsked,
17625        )
17626        .body
17627    }
17628
17629    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17630
17631    #[test]
17632    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17633        let mut state = review_state(&["feat: x"]);
17634        state.reviews = vec![
17635            round_with_notes(1, Some(SANDBOXED)),
17636            passing_round(2, Some(FINAL)),
17637        ];
17638        ok_gate(&mut state, Some(FINAL));
17639        let body = body_at(&state);
17640        assert!(!body.contains("Handled the PATH case."), "{body}");
17641        assert!(!body.contains("could not commit"), "{body}");
17642        assert!(!body.contains("could not run"), "{body}");
17643        assert!(
17644            body.contains("The fix is committed on the branch"),
17645            "{body}"
17646        );
17647        assert!(!body.contains("sandbox"), "{body}");
17648        assert!(
17649            body.contains("Tests (`verify.e2e`): passed on the final commit"),
17650            "{body}"
17651        );
17652        assert!(
17653            body.contains("Gate (`verify.gate`): passed on the final commit"),
17654            "{body}"
17655        );
17656        // The raw report is untouched in the record.
17657        assert!(
17658            state.reviews[0]
17659                .fix
17660                .as_ref()
17661                .unwrap()
17662                .notes
17663                .contains("could not commit")
17664        );
17665    }
17666
17667    #[test]
17668    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17669        let mut state = review_state(&["feat: x"]);
17670        let mut r1 = round_with_notes(1, Some(SANDBOXED));
17671        r1.fix.as_mut().unwrap().committed = false;
17672        let mut r2 = passing_round(2, Some(FINAL));
17673        r2.e2e[0].code = Some(1);
17674        state.reviews = vec![r1, r2];
17675        let body = body_at(&state);
17676        assert!(!body.contains("could not commit"), "{body}");
17677        assert!(!body.contains("could not run"), "{body}");
17678        assert!(body.contains("This round produced no commit"), "{body}");
17679        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17680        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17681        assert!(!body.contains("passed"), "{body}");
17682    }
17683
17684    #[test]
17685    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17686        let mut state = review_state(&["feat: x"]);
17687        state.reviews = vec![
17688            round_with_notes(1, Some(SANDBOXED)),
17689            passing_round(2, Some("0123456789abcdef")),
17690        ];
17691        ok_gate(&mut state, Some("0123456789abcdef"));
17692        let body = body_at(&state);
17693        assert!(!body.contains("HTTP tests could not run"), "{body}");
17694        // The commit claim is still settled by the structured record.
17695        assert!(!body.contains("could not commit"), "{body}");
17696        assert!(
17697            body.contains("passed on an earlier commit (`0123456`)"),
17698            "{body}"
17699        );
17700        assert!(!body.contains("passed on the final commit"), "{body}");
17701
17702        // A gate recorded before `gate_head` existed is "not recorded".
17703        ok_gate(&mut state, None);
17704        let body = body_at(&state);
17705        assert!(
17706            body.contains("passed on a commit that was not recorded"),
17707            "{body}"
17708        );
17709    }
17710
17711    #[test]
17712    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17713        let state = state_with_summary(
17714            "add retries to the client",
17715            "TITLE: feat: retries\n\nDid it.",
17716        );
17717        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17718        assert_eq!(m.title, "feat: retries");
17719        assert!(
17720            m.body.contains("<summary>Original task</summary>"),
17721            "{}",
17722            m.body
17723        );
17724        assert!(!m.body.contains("Commits under review"), "{}", m.body);
17725        assert!(
17726            m.body
17727                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17728            "{}",
17729            m.body
17730        );
17731    }
17732
17733    #[test]
17734    fn pr_message_review_bounds_a_long_english_subject() {
17735        let long = format!("feat: {}", "word ".repeat(100));
17736        let m = pr_message(&review_state(&[&long]), 'A');
17737        assert!(m.title.starts_with("feat: word"), "{}", m.title);
17738        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17739    }
17740
17741    #[test]
17742    fn pr_message_implementation_run_is_unchanged_by_review_support() {
17743        let state = state_with_summary("add retries\n\ndetails", "- did some things");
17744        let m = pr_message(&state, 'A');
17745        assert_eq!(m.title, "add retries");
17746        assert!(m.body.contains("<summary>Original task</summary>"));
17747        assert!(!m.body.contains("Commits under review"));
17748        assert_eq!(landing_subject_source(&state), state.instruction);
17749    }
17750
17751    #[test]
17752    fn review_run_squash_subject_is_the_change_not_the_prompt() {
17753        let state = review_state(&["feat: the change", "fix: typo"]);
17754        let source = landing_subject_source(&state);
17755        assert_eq!(land::merge_subject("", &source), "feat: the change");
17756        assert_eq!(
17757            land::merge_subject("magi: candidate A (uncommitted work)", &source),
17758            "feat: the change"
17759        );
17760        // An operator's rename still wins.
17761        assert_eq!(
17762            land::merge_subject("feat: renamed by hand", &source),
17763            "feat: renamed by hand"
17764        );
17765        let blank = review_state(&["日本語"]);
17766        assert!(
17767            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17768        );
17769    }
17770
17771    #[test]
17772    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17773        let state = review_state(&["feat: the change"]);
17774        let source = landing_subject_source(&state);
17775        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17776        assert_eq!(
17777            land::merge_subject(landing_title(&state, old), &source),
17778            "feat: the change"
17779        );
17780        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17781        let task = state_with_summary("add retries", "");
17782        assert_eq!(landing_title(&task, old), old);
17783    }
17784
17785    #[test]
17786    fn pr_message_describes_the_change_not_the_task() {
17787        let state = state_with_summary(
17788            "今回やってほしいこと: results projector を直す",
17789            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17790        );
17791        let m = pr_message(&state, 'A');
17792        assert_eq!(m.title, "fix(web): batch the runs list reads");
17793        assert!(
17794            m.body.starts_with("## Summary\n\n- reads run.json once"),
17795            "{}",
17796            m.body
17797        );
17798        assert!(!m.body.contains("TITLE:"), "{}", m.body);
17799        let task_at = m.body.find("今回やってほしいこと").unwrap();
17800        let details_at = m.body.find("<details>").unwrap();
17801        assert!(
17802            details_at < task_at,
17803            "the task lives inside <details>: {}",
17804            m.body
17805        );
17806        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17807        assert!(m.body.contains("magi:candidate-a"));
17808    }
17809
17810    #[test]
17811    fn pr_message_falls_back_to_the_task_without_a_title_line() {
17812        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17813        let m = pr_message(&state, 'A');
17814        assert_eq!(m.title, "add retries");
17815        assert!(
17816            m.body.contains("## Summary\n\n- did some things"),
17817            "{}",
17818            m.body
17819        );
17820
17821        let none = RunState::new(
17822            PathBuf::from("/repo"),
17823            "main".to_owned(),
17824            "abc1234".to_owned(),
17825            "add retries".to_owned(),
17826            Config::default(),
17827        );
17828        let m = pr_message(&none, 'A');
17829        assert_eq!(m.title, "add retries");
17830        assert!(!m.body.contains("## Summary"), "{}", m.body);
17831    }
17832
17833    #[test]
17834    fn pr_message_refuses_the_candidate_commit_subject() {
17835        for bad in [
17836            "TITLE: magi: candidate A (uncommitted work)",
17837            "TITLE: chore: stuff (uncommitted work)",
17838            "TITLE:   ",
17839        ] {
17840            let state = state_with_summary("add retries", bad);
17841            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17842        }
17843    }
17844
17845    #[test]
17846    fn pr_message_bounds_a_very_long_task_and_title() {
17847        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17848        let state = state_with_summary(&long, "- nothing");
17849        let m = pr_message(&state, 'A');
17850        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17851        assert!(!m.title.contains('\n'));
17852
17853        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17854        let m = pr_message(&state, 'A');
17855        assert!(m.title.starts_with("feat: "));
17856        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17857        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17858    }
17859
17860    fn long_title_of(instruction: &str) -> String {
17861        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17862    }
17863
17864    #[test]
17865    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17866        let first = "Make the landing path keep a readable title for long tasks";
17867        let line = format!(
17868            "{first}. {}",
17869            "And then keep going with more words ".repeat(20)
17870        );
17871        let t = long_title_of(&line);
17872        assert_eq!(t, first);
17873        assert!(!t.starts_with("chore: land"));
17874    }
17875
17876    #[test]
17877    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17878        let line = "word ".repeat(200);
17879        let t = long_title_of(&line);
17880        assert!(t.ends_with("word..."), "{t}");
17881        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17882    }
17883
17884    #[test]
17885    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17886        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17887            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17888        }
17889    }
17890
17891    #[test]
17892    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17893        let mut state = state_with_summary("再試行を追加する", "- nothing");
17894        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17895        state.config.graph.github_language = "ja".into();
17896        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17897    }
17898
17899    #[test]
17900    fn pr_message_title_limit_is_exact() {
17901        let at = "a".repeat(PR_TITLE_MAX);
17902        assert_eq!(long_title_of(&at), at);
17903        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17904        assert!(over.ends_with("..."), "{over}");
17905        assert_eq!(over.chars().count(), PR_TITLE_MAX);
17906    }
17907
17908    #[test]
17909    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17910        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17911        let t = long_title_of(&line);
17912        assert!(t.ends_with("..."), "{t}");
17913        assert!(t.is_ascii(), "{t}");
17914    }
17915
17916    #[test]
17917    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17918        let line = format!(
17919            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17920            "plus more filler words ".repeat(20)
17921        );
17922        let t = long_title_of(&line);
17923        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17924    }
17925
17926    #[test]
17927    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17928        let line = format!("\n\n# {}", "title words ".repeat(40));
17929        let state = state_with_summary(&line, "- nothing");
17930        let m = pr_message(&state, 'A');
17931        assert!(m.title.starts_with("title words"), "{}", m.title);
17932        assert_eq!(
17933            land::merge_subject(&m.title, &landing_subject_source(&state)),
17934            m.title
17935        );
17936        // An operator's rename wins untouched.
17937        assert_eq!(
17938            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17939            "feat: renamed by hand"
17940        );
17941    }
17942
17943    #[tokio::test]
17944    async fn github_text_rewrite_is_bounded_and_falls_back() {
17945        crate::run::pin_test_home();
17946        for (reply, accepted) in [
17947            (
17948                "TITLE: fix: retries\nAdd retries for failed requests.",
17949                true,
17950            ),
17951            (
17952                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17953                false,
17954            ),
17955            (
17956                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17957                true,
17958            ),
17959            (
17960                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17961                true,
17962            ),
17963            (
17964                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17965                true,
17966            ),
17967            (
17968                "TITLE: fix: retries\n日本語の説明です。 token=secret",
17969                false,
17970            ),
17971            ("TITLE: fix: retries\n", false),
17972        ] {
17973            let dir = tempfile::tempdir().unwrap();
17974            let mut runner = runner_at(RunStatus::Gating);
17975            runner.state = state_with_summary(
17976                "add retries",
17977                "TITLE: fix: retries\n日本語の説明を書きます。",
17978            );
17979            runner.state.repo = dir.path().to_owned();
17980            runner.state.candidates[0].worktree = dir.path().to_owned();
17981            let mut author = spec("alpha");
17982            author.command = vec![
17983                "sh".into(),
17984                "-c".into(),
17985                "printf '%s' \"$REWRITE_REPLY\"".into(),
17986            ];
17987            author.env.insert("REWRITE_REPLY".into(), reply.into());
17988            runner.state.config.agents = vec![author];
17989            seed_public_summary(&mut runner);
17990            let winner = runner.state.candidates[0].clone();
17991            let first = runner
17992                .guarded_pr_message(&winner, None, true)
17993                .await
17994                .unwrap();
17995            let message = if accepted {
17996                first.expect("a passing rewrite never asks")
17997            } else {
17998                // Withheld: the run parks on a question and posts nothing yet.
17999                assert!(first.is_none());
18000                assert!(runner.state.parked);
18001                let store = ask::Questions::open();
18002                let asked: Vec<_> = store
18003                    .list()
18004                    .into_iter()
18005                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18006                    .collect();
18007                assert_eq!(asked.len(), 1);
18008                assert!(!asked[0].detail.contains("secret"));
18009                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18010                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18011                assert_eq!(
18012                    asked[0]
18013                        .choices
18014                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18015                    rewrite.is_some()
18016                );
18017                // Silence: the question lapses and the neutral text is used,
18018                // without a second rewrite call or a second question.
18019                store
18020                    .update(&asked[0].id, |q| {
18021                        q.abandon("test");
18022                        Ok(())
18023                    })
18024                    .unwrap();
18025                let again = runner
18026                    .guarded_pr_message(&winner, None, true)
18027                    .await
18028                    .unwrap()
18029                    .expect("an abandoned question falls back");
18030                let n = store
18031                    .list()
18032                    .into_iter()
18033                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18034                    .count();
18035                assert_eq!(n, 1, "asked once per rejected text");
18036                again
18037            };
18038            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18039            assert!(!message.body.contains("secret"));
18040            assert!(!message.body.contains("prefix"));
18041            assert!(!message.title.contains("secret"));
18042            assert_eq!(
18043                message.body.contains("Add retries for failed requests."),
18044                accepted
18045            );
18046            assert_eq!(
18047                runner.state.seats["github-text-rewrite"].turns, 1,
18048                "only one rewrite invocation"
18049            );
18050            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18051            if accepted {
18052                assert!(runner.state.github_text.is_none());
18053                assert!(!runner.state.parked);
18054                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18055                assert!(message.body.contains("## Final verification"));
18056                assert!(message.body.contains("Evidence: final diff"));
18057                assert!(
18058                    message
18059                        .body
18060                        .contains(&format!("magi:run/{}", runner.state.id))
18061                );
18062            }
18063            if !accepted {
18064                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18065                assert!(
18066                    message
18067                        .body
18068                        .contains(&format!("magi:run/{}", runner.state.id))
18069                );
18070            }
18071        }
18072    }
18073
18074    fn scripted(id: &str, reply: &str) -> AgentSpec {
18075        let mut a = spec(id);
18076        a.command = vec![
18077            "sh".into(),
18078            "-c".into(),
18079            "printf '%s' \"$REWRITE_REPLY\"".into(),
18080        ];
18081        a.env.insert("REWRITE_REPLY".into(), reply.into());
18082        a
18083    }
18084
18085    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18086    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18087
18088    /// Gate tests inject generated public output, not internal reports.
18089    fn seed_public_summary(runner: &mut Runner) {
18090        let repo = &runner.state.repo;
18091        let g = |args: &[&str]| {
18092            let out = std::process::Command::new("git")
18093                .args(args)
18094                .current_dir(repo)
18095                .quiet()
18096                .output()
18097                .unwrap();
18098            assert!(out.status.success(), "git {args:?}");
18099            String::from_utf8_lossy(&out.stdout).trim().to_owned()
18100        };
18101        g(&["init", "-q", "-b", "main"]);
18102        g(&[
18103            "-c",
18104            "user.name=Test",
18105            "-c",
18106            "user.email=test@example.invalid",
18107            "commit",
18108            "-q",
18109            "--allow-empty",
18110            "-m",
18111            "feat: fixture",
18112        ]);
18113        let head = g(&["rev-parse", "HEAD"]);
18114        g(&["update-ref", "refs/remotes/origin/main", &head]);
18115        let evidence = crate::pr_outcome::summary_evidence(
18116            &runner.state,
18117            crate::pr_outcome::FinalHead::At(&head),
18118        );
18119        let generated = &runner.state.candidates[0].summary;
18120        runner.state.change_summary = Some(crate::run::ChangeSummary {
18121            base: head.clone(),
18122            head,
18123            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18124            title: summary_title(generated).unwrap(),
18125            body: summary_without_title(generated),
18126        });
18127    }
18128
18129    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18130        let mut runner = runner_at(RunStatus::Gating);
18131        runner.state = state_with_summary("add retries", JA_SUMMARY);
18132        if review_only {
18133            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18134        }
18135        runner.state.repo = dir.to_owned();
18136        runner.state.candidates[0].worktree = dir.to_owned();
18137        runner.state.config.agents = agents;
18138        seed_public_summary(&mut runner);
18139        runner
18140    }
18141
18142    fn gate_questions(runner: &Runner) -> usize {
18143        ask::Questions::open()
18144            .list()
18145            .into_iter()
18146            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18147            .count()
18148    }
18149
18150    #[tokio::test]
18151    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18152        crate::run::pin_test_home();
18153        let dir = tempfile::tempdir().unwrap();
18154        let mut runner = rewrite_runner(
18155            dir.path(),
18156            true,
18157            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18158        );
18159        runner.state.config.roles.language_judge =
18160            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18161        let winner = runner.state.candidates[0].clone();
18162        let msg = runner
18163            .guarded_pr_message(&winner, None, true)
18164            .await
18165            .unwrap()
18166            .expect("an English rewrite posts without asking");
18167        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18168        assert!(msg.body.contains("<details>kept</details>"));
18169        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18170        assert_eq!(gate_questions(&runner), 0);
18171        assert!(runner.state.github_text.is_none());
18172        assert!(!runner.state.seats.contains_key("impl-A"));
18173        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18174    }
18175
18176    #[tokio::test]
18177    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18178        crate::run::pin_test_home();
18179        let dir = tempfile::tempdir().unwrap();
18180        let mut runner = rewrite_runner(
18181            dir.path(),
18182            false,
18183            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18184        );
18185        runner.state.config.roles.language_judge =
18186            Some(crate::config::AgentChoice::One("beta".into()));
18187        runner.state.seats.insert(
18188            "impl-A".into(),
18189            SeatState::new("impl-A", "alpha", runner.state.seed),
18190        );
18191        let winner = runner.state.candidates[0].clone();
18192        let msg = runner
18193            .guarded_pr_message(&winner, None, true)
18194            .await
18195            .unwrap()
18196            .expect("rewritten");
18197        assert!(msg.body.contains("<details>kept</details>"));
18198        assert_eq!(runner.state.seats["impl-A"].turns, 0);
18199        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18200        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18201    }
18202
18203    #[tokio::test]
18204    async fn rewrite_reuses_a_resumable_implementer_session() {
18205        crate::run::pin_test_home();
18206        let dir = tempfile::tempdir().unwrap();
18207        let mut runner = rewrite_runner(
18208            dir.path(),
18209            false,
18210            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18211        );
18212        runner.state.config.roles.language_judge =
18213            Some(crate::config::AgentChoice::One("beta".into()));
18214        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18215        seat.turns = 1;
18216        runner.state.seats.insert("impl-A".into(), seat);
18217        let winner = runner.state.candidates[0].clone();
18218        runner
18219            .guarded_pr_message(&winner, None, true)
18220            .await
18221            .unwrap()
18222            .expect("rewritten");
18223        assert_eq!(runner.state.seats["impl-A"].turns, 2);
18224        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18225    }
18226
18227    #[tokio::test]
18228    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18229        crate::run::pin_test_home();
18230        let dir = tempfile::tempdir().unwrap();
18231        let mut broken = spec("bad");
18232        broken.command = vec!["false".into()];
18233        let mut runner = rewrite_runner(
18234            dir.path(),
18235            true,
18236            vec![
18237                broken,
18238                scripted("noformat", "Add retries."),
18239                scripted("good", EN_REWRITE),
18240            ],
18241        );
18242        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18243            "bad".into(),
18244            "noformat".into(),
18245            "good".into(),
18246        ]));
18247        let winner = runner.state.candidates[0].clone();
18248        let msg = runner
18249            .guarded_pr_message(&winner, None, true)
18250            .await
18251            .unwrap()
18252            .expect("the third agent rewrites");
18253        assert!(msg.body.contains("`src/net.rs`"));
18254        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18255        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18256    }
18257
18258    #[tokio::test]
18259    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18260        crate::run::pin_test_home();
18261        // (agents, role): still Japanese; no TITLE; nothing resolvable.
18262        for (agents, role, saved) in [
18263            (
18264                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18265                Some("beta"),
18266                true,
18267            ),
18268            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18269            (Vec::new(), None, false),
18270        ] {
18271            let dir = tempfile::tempdir().unwrap();
18272            let mut runner = rewrite_runner(dir.path(), true, agents);
18273            runner.state.config.roles.language_judge =
18274                role.map(|r| crate::config::AgentChoice::One(r.into()));
18275            let winner = runner.state.candidates[0].clone();
18276            let first = runner
18277                .guarded_pr_message(&winner, None, true)
18278                .await
18279                .unwrap();
18280            assert!(first.is_none(), "withheld text parks the run");
18281            assert!(runner.state.parked);
18282            assert_eq!(gate_questions(&runner), 1);
18283            let g = runner.state.github_text.as_ref().unwrap();
18284            assert_eq!(g.rewrite.is_some(), saved);
18285            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18286        }
18287    }
18288
18289    #[tokio::test]
18290    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18291        crate::run::pin_test_home();
18292        let dir = tempfile::tempdir().unwrap();
18293        let mut runner = runner_at(RunStatus::Gating);
18294        runner.state =
18295            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18296        runner.state.repo = dir.path().to_owned();
18297        runner.state.candidates[0].worktree = dir.path().to_owned();
18298        let mut author = spec("alpha");
18299        author.command = vec![
18300            "sh".into(),
18301            "-c".into(),
18302            "printf '%s' \"$REWRITE_REPLY\"".into(),
18303        ];
18304        author.env.insert(
18305            "REWRITE_REPLY".into(),
18306            format!(
18307                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18308                "English sentence. ".repeat(600)
18309            ),
18310        );
18311        runner.state.config.agents = vec![author];
18312        seed_public_summary(&mut runner);
18313        let winner = runner.state.candidates[0].clone();
18314        assert!(
18315            runner
18316                .guarded_pr_message(&winner, None, true)
18317                .await
18318                .unwrap()
18319                .is_none()
18320        );
18321        let record = runner.state.github_text.clone().unwrap();
18322        let rewrite = record.rewrite.clone().unwrap();
18323        assert_eq!(rewrite.failed, ["body-language"]);
18324        assert!(
18325            rewrite
18326                .body
18327                .ends_with(&format!("magi:run/{}", runner.state.id))
18328        );
18329        let store = ask::Questions::open();
18330        let q = store.get(record.question.as_ref().unwrap()).unwrap();
18331        assert!(
18332            q.choices
18333                .contains(&crate::github_text::USE_REWRITE.to_owned())
18334        );
18335        for label in [
18336            "ORIGINAL",
18337            "FALLBACK",
18338            "REWRITE",
18339            "Rules still failed: body-language",
18340            "truncated",
18341        ] {
18342            assert!(q.detail.contains(label), "{label}");
18343        }
18344        let artifact = std::fs::read_to_string(crate::run::artifact_path(
18345            &runner.state,
18346            &crate::github_text::artifact_name(&record.fingerprint),
18347        ))
18348        .unwrap();
18349        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18350        // Restart from serialized state, then retry after a bad owner title.
18351        runner.state =
18352            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18353        store
18354            .update(&q.id, |q| {
18355                q.say("まだ日本語")?;
18356                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18357            })
18358            .unwrap();
18359        assert!(
18360            runner
18361                .guarded_pr_message(&winner, None, true)
18362                .await
18363                .unwrap()
18364                .is_none()
18365        );
18366        let retry = runner.state.github_text.as_ref().unwrap();
18367        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18368        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18369        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18370        assert!(
18371            q.posting_comparison
18372                .as_ref()
18373                .unwrap()
18374                .matches_detail(&q.detail)
18375        );
18376        store
18377            .update(&q.id, |q| {
18378                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18379            })
18380            .unwrap();
18381        let message = runner
18382            .guarded_pr_message(&winner, None, true)
18383            .await
18384            .unwrap()
18385            .unwrap();
18386        assert_eq!(message.title, rewrite.title);
18387        assert_eq!(message.body, rewrite.body);
18388        assert_eq!(
18389            runner.state.github_text.as_ref().unwrap().choice,
18390            Some(GithubTextChoice::Rewrite)
18391        );
18392        let again = runner
18393            .guarded_pr_message(&winner, None, true)
18394            .await
18395            .unwrap()
18396            .unwrap();
18397        assert_eq!(again.title, message.title);
18398        assert_eq!(again.body, message.body);
18399        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18400        assert!(
18401            !runner
18402                .state
18403                .events
18404                .iter()
18405                .any(|e| e.message.contains(&rewrite.title)
18406                    || e.message.contains("English sentence."))
18407        );
18408    }
18409
18410    #[tokio::test]
18411    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18412        crate::run::pin_test_home();
18413        let dir = tempfile::tempdir().unwrap();
18414        let mut runner = runner_at(RunStatus::Gating);
18415        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18416        runner.state.repo = dir.path().to_owned();
18417        runner.state.candidates[0].worktree = dir.path().to_owned();
18418        seed_public_summary(&mut runner);
18419        let winner = runner.state.candidates[0].clone();
18420        assert!(
18421            runner
18422                .guarded_pr_message(&winner, None, true)
18423                .await
18424                .unwrap()
18425                .is_none()
18426        );
18427        let record = runner.state.github_text.as_ref().unwrap();
18428        assert!(record.rewrite.is_none());
18429        // Records written before the additive field deserialize with no rewrite.
18430        let mut old = serde_json::to_value(record).unwrap();
18431        old.as_object_mut().unwrap().remove("rewrite");
18432        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18433        assert!(old.rewrite.is_none());
18434        let id = record.question.clone().unwrap();
18435        ask::Questions::open()
18436            .update(&id, |q| {
18437                q.choices.push(crate::github_text::USE_REWRITE.into());
18438                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18439            })
18440            .unwrap();
18441        let message = runner
18442            .guarded_pr_message(&winner, None, true)
18443            .await
18444            .unwrap()
18445            .unwrap();
18446        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18447        assert_eq!(
18448            runner.state.github_text.as_ref().unwrap().choice,
18449            Some(GithubTextChoice::Fallback)
18450        );
18451    }
18452
18453    #[tokio::test]
18454    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18455        crate::run::pin_test_home();
18456        let dir = tempfile::tempdir().unwrap();
18457        let mut runner = runner_at(RunStatus::Gating);
18458        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18459        runner.state.repo = dir.path().to_owned();
18460        runner.state.candidates[0].worktree = dir.path().to_owned();
18461        seed_public_summary(&mut runner);
18462        let winner = runner.state.candidates[0].clone();
18463        assert!(
18464            runner
18465                .guarded_pr_message(&winner, None, true)
18466                .await
18467                .unwrap()
18468                .is_none()
18469        );
18470        let store = ask::Questions::open();
18471        let find = |run: &str| {
18472            store
18473                .list()
18474                .into_iter()
18475                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18476                .unwrap()
18477        };
18478        let run = runner.state.id.clone();
18479        let q = find(&run);
18480        let fp = runner
18481            .state
18482            .github_text
18483            .as_ref()
18484            .unwrap()
18485            .fingerprint
18486            .clone();
18487        let name = crate::github_text::artifact_name(&fp);
18488        let path = crate::run::artifact_path(&runner.state, &name);
18489        assert!(q.detail.contains(&path.display().to_string()));
18490        assert!(q.detail.contains("title-language: title"));
18491        let saved = std::fs::read_to_string(&path).expect("artifact written");
18492        assert!(saved.contains("日本語のタイトル"));
18493        assert!(q.cwd.is_none());
18494        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18495        // A bad replacement is refused and asked about once more.
18496        let say = |id: &str, text: &str, choice: &str| {
18497            store
18498                .update(id, |q| {
18499                    q.thread.push(ask::Turn {
18500                        who: ask::Who::Operator,
18501                        body: text.to_owned(),
18502                        at: jiff::Timestamp::now(),
18503                        note: None,
18504                    });
18505                    q.answer(ask::Answer::Choice(choice.to_owned()))
18506                })
18507                .unwrap();
18508        };
18509        say(&q.id, "まだ日本語", "use my text");
18510        assert!(
18511            runner
18512                .guarded_pr_message(&winner, None, true)
18513                .await
18514                .unwrap()
18515                .is_none()
18516        );
18517        let second = find(&run);
18518        assert_ne!(second.id, q.id);
18519        assert!(second.detail.contains("did not pass"));
18520        // The retry still compares the original text; the rejected
18521        // replacement is named by category only.
18522        assert!(second.detail.contains("日本語のタイトル"));
18523        assert!(!second.detail.contains("まだ日本語"));
18524        assert!(second.detail.contains(&path.display().to_string()));
18525        say(&second.id, "fix: retry failed requests", "use my text");
18526        let message = runner
18527            .guarded_pr_message(&winner, None, true)
18528            .await
18529            .unwrap()
18530            .expect("a vetted replacement is posted");
18531        assert_eq!(message.title, "fix: retry failed requests");
18532        assert_eq!(
18533            runner
18534                .state
18535                .github_text
18536                .as_ref()
18537                .unwrap()
18538                .chosen_title
18539                .as_deref(),
18540            Some("fix: retry failed requests")
18541        );
18542        assert!(runner.state.github_text.as_ref().unwrap().resolved);
18543        // Resumed again: the saved decision is applied, nothing is re-asked.
18544        let again = runner
18545            .guarded_pr_message(&winner, None, true)
18546            .await
18547            .unwrap()
18548            .unwrap();
18549        assert_eq!(again.title, message.title);
18550        assert_eq!(
18551            store
18552                .list()
18553                .into_iter()
18554                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18555                .count(),
18556            2
18557        );
18558    }
18559
18560    #[test]
18561    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18562        // What magi itself writes stays English under any configured language,
18563        // so a future localisation of these headings fails here. (The agents'
18564        // own text is also checked by the posting gate.)
18565        let mut state = state_with_summary(
18566            "add retries",
18567            "TITLE: fix(web): batch reads\n- reads run.json once",
18568        );
18569        state.config.graph.language = "ja".to_owned();
18570        let m = pr_message(&state, 'A');
18571        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18572        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18573
18574        // The task is the operator's own text: it goes in untouched, and the
18575        // fallback title (no summary) may be in its language too.
18576        let task = "今回やってほしいこと: results projector を直す";
18577        let mut state = state_with_summary(task, "- no title line");
18578        state.config.graph.language = "ja".to_owned();
18579        let m = pr_message(&state, 'A');
18580        assert_eq!(
18581            m.title,
18582            format!("chore: land candidate A of run {}", state.id)
18583        );
18584        assert!(
18585            m.body.contains(&format!(
18586                "<summary>Original task</summary>\n\n{task}\n\n</details>"
18587            )),
18588            "{}",
18589            m.body
18590        );
18591    }
18592
18593    #[test]
18594    fn pr_message_scrubs_home_paths_and_addresses() {
18595        let state = state_with_summary(
18596            "fix it in /Users/someone/src/x",
18597            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18598        );
18599        let m = pr_message(&state, 'A');
18600        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18601            assert!(!m.body.contains(leak), "{}", m.body);
18602        }
18603        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18604    }
18605
18606    #[test]
18607    fn pr_message_survives_a_task_that_closes_details() {
18608        let state = state_with_summary("a </details> b", "TITLE: fix: x");
18609        let m = pr_message(&state, 'A');
18610        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18611    }
18612
18613    #[test]
18614    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18615        let cmd = manual_merge_command(
18616            MergeStyle::Squash,
18617            Path::new("/repo"),
18618            "b",
18619            "fix: \"quoted\" $(x) `y`\n\nbody",
18620        );
18621        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18622    }
18623
18624    #[test]
18625    fn manual_merge_command_matches_the_configured_style() {
18626        let repo = Path::new("/repo");
18627        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18628
18629        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18630        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18631
18632        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18633        assert_eq!(
18634            squash,
18635            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18636             \"Merge magi run 0832 (candidate A)\""
18637        );
18638
18639        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18640        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18641    }
18642
18643    #[test]
18644    fn a_nudge_gets_a_quarter_of_the_budget() {
18645        // The judge and implement budgets magi ships with.
18646        assert_eq!(retry_budget(secs(1200), true), secs(300));
18647        assert_eq!(retry_budget(secs(3600), true), secs(900));
18648    }
18649
18650    #[test]
18651    fn a_resent_prompt_keeps_the_whole_budget() {
18652        // The seat kept no context, so the retry is the original job again and
18653        // shortening it would only guarantee a second failure.
18654        assert_eq!(retry_budget(secs(1200), false), secs(1200));
18655        assert_eq!(retry_budget(secs(60), false), secs(60));
18656    }
18657
18658    #[test]
18659    fn the_floor_never_exceeds_the_original_budget() {
18660        // A short configured timeout must not be *raised* by the floor: the
18661        // operator asked for a bound, and a retry may not outlast the attempt
18662        // it is retrying.
18663        assert_eq!(retry_budget(secs(60), true), secs(60));
18664        assert_eq!(retry_budget(secs(480), true), secs(120));
18665        assert_eq!(retry_budget(secs(0), true), secs(0));
18666    }
18667
18668    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18669        agent::CommandEvidence {
18670            id: "item1".to_owned(),
18671            description: "cargo test".to_owned(),
18672            exit_code,
18673            result_summary: String::new(),
18674            source: "codex".to_owned(),
18675        }
18676    }
18677
18678    #[test]
18679    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18680        // No evidence is not the same fact as unconfirmed evidence: a
18681        // backend with no adapter, or a reply that ran no commands at all,
18682        // must not be misread as carrying a dangling job.
18683        assert!(!has_unconfirmed_command(&[]));
18684    }
18685
18686    #[test]
18687    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18688        // Deliberately not a check on the exit code's *value*: a fixer
18689        // legitimately runs something that fails mid-iteration before it
18690        // succeeds, and that must never by itself reopen a valid report.
18691        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18692        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18693        assert!(!has_unconfirmed_command(&[
18694            evidence(Some(0)),
18695            evidence(Some(101))
18696        ]));
18697    }
18698
18699    #[test]
18700    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18701        assert!(has_unconfirmed_command(&[
18702            evidence(Some(0)),
18703            evidence(None)
18704        ]));
18705    }
18706
18707    #[test]
18708    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18709        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18710        assert_eq!(
18711            verified_noop_claim(true, &[], text).as_deref(),
18712            Some("already fixed by b32cfc4, on main.")
18713        );
18714    }
18715
18716    #[test]
18717    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18718        // A timeout or a bad exit code reads as the ordinary loss it is,
18719        // whatever the reply's own prose claims.
18720        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18721        assert!(verified_noop_claim(false, &[], text).is_none());
18722    }
18723
18724    #[test]
18725    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18726        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18727        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18728        // A confirmed command alongside the marker is fine.
18729        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18730    }
18731
18732    #[test]
18733    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18734        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18735    }
18736
18737    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
18738    /// pair, in order, labelled A, B, C, ...
18739    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18740        runner.state.candidates = shape
18741            .iter()
18742            .enumerate()
18743            .map(|(i, &(empty, verified))| Candidate {
18744                index: i,
18745                label: (b'A' + i as u8) as char,
18746                agent: "sonnet".to_owned(),
18747                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18748                worktree: PathBuf::from(format!("/wt/{i}")),
18749                summary: String::new(),
18750                stat: String::new(),
18751                files: 0,
18752                commits: 0,
18753                empty,
18754                failed: None,
18755                verified_noop: verified.map(str::to_owned),
18756                duration_ms: 0,
18757                folded: false,
18758            })
18759            .collect();
18760    }
18761
18762    #[test]
18763    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18764        ask_test_home();
18765        let mut runner = runner_at(RunStatus::Implementing);
18766        set_candidates(
18767            &mut runner,
18768            &[
18769                (true, Some("already on main at b32cfc4")),
18770                (true, Some("same fix, see the existing test")),
18771            ],
18772        );
18773
18774        runner
18775            .after_implement()
18776            .expect("a verified no-op is not an error");
18777
18778        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18779    }
18780
18781    #[test]
18782    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18783        ask_test_home();
18784        let mut runner = runner_at(RunStatus::Implementing);
18785        // Candidate A declares a verified no-op; candidate B simply wrote
18786        // nothing and said nothing about why. One candidate's claim is not
18787        // the whole run's agreement.
18788        set_candidates(
18789            &mut runner,
18790            &[(true, Some("already on main at b32cfc4")), (true, None)],
18791        );
18792
18793        let err = runner
18794            .after_implement()
18795            .expect_err("an unverified empty candidate must still fail the run");
18796
18797        assert!(
18798            err.to_string().contains("no candidate produced a change"),
18799            "{err}"
18800        );
18801        assert_eq!(runner.state.status, RunStatus::Failed);
18802    }
18803
18804    #[test]
18805    fn after_implement_still_fails_an_ordinary_all_empty_run() {
18806        ask_test_home();
18807        let mut runner = runner_at(RunStatus::Implementing);
18808        set_candidates(&mut runner, &[(true, None), (true, None)]);
18809
18810        let err = runner
18811            .after_implement()
18812            .expect_err("no candidate declared anything; this is an ordinary failure");
18813
18814        assert!(
18815            err.to_string().contains("no candidate produced a change"),
18816            "{err}"
18817        );
18818        assert_eq!(runner.state.status, RunStatus::Failed);
18819    }
18820}
18821
18822/// A graft rollback whose restore could not be confirmed (see
18823/// [`Runner::graft`]).
18824#[derive(Debug)]
18825struct GraftRestoreFailed(String);
18826
18827impl std::fmt::Display for GraftRestoreFailed {
18828    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18829        write!(
18830            f,
18831            "could not restore the winner to its pre-graft commit: {}",
18832            self.0
18833        )
18834    }
18835}
18836
18837impl std::error::Error for GraftRestoreFailed {}
18838
18839/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
18840/// not the agent: they never draw on an agent's own timeout, and running out of
18841/// them never reads as an agent failure.
18842#[cfg(test)]
18843mod resource_wait_tests {
18844    use super::*;
18845    use std::collections::BTreeMap;
18846
18847    fn sh(id: &str, script: &str) -> AgentSpec {
18848        AgentSpec {
18849            id: id.to_owned(),
18850            kind: crate::config::AgentKind::Command,
18851            model: None,
18852            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18853            extra_args: Vec::new(),
18854            env: BTreeMap::new(),
18855            prompt_delivery: None,
18856        }
18857    }
18858
18859    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18860        SeatJob {
18861            seat: SeatState::new("impl-A", &spec.id, 1),
18862            spec,
18863            cwd: dir.to_path_buf(),
18864            prompt: "do it".to_owned(),
18865            timeout,
18866            allow_write: true,
18867            sessions: false,
18868            artifacts: dir.join("art"),
18869            stem: "impl-A".to_owned(),
18870            handover: None,
18871        }
18872    }
18873
18874    fn state(resource_wait_secs: u64) -> RunState {
18875        let mut config = Config::default();
18876        config.graph.timeout_resource_wait = resource_wait_secs;
18877        RunState::new(
18878            PathBuf::from("/repo"),
18879            "main".to_owned(),
18880            "abc1234".to_owned(),
18881            "task".to_owned(),
18882            config,
18883        )
18884    }
18885
18886    fn hold(cache: &Path) -> crate::cache::Guard {
18887        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18888        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18889            crate::cache::AcquireOutcome::Acquired(g) => g,
18890            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18891        }
18892    }
18893
18894    async fn run(
18895        jobs: Vec<SeatJob>,
18896        sem: Arc<Semaphore>,
18897        cache: Option<&Path>,
18898        st: &mut RunState,
18899    ) -> Vec<(usize, SeatState, AgentOutcome)> {
18900        let prompts = Prompts::default();
18901        let ctx = WaveCtx {
18902            run: "r",
18903            node: "implement",
18904            prompts: &prompts,
18905            cache,
18906            round: None,
18907            carry_seats: false,
18908            pauses: None,
18909        };
18910        wave(jobs, sem, &ctx, st, 0).await
18911    }
18912
18913    #[tokio::test]
18914    async fn verification_cache_wait_preserves_the_command_budget() {
18915        crate::run::pin_test_home();
18916        let tmp = tempfile::tempdir().unwrap();
18917        let cache = tmp.path().join("target");
18918        let guard = hold(&cache);
18919        let release = tokio::spawn(async move {
18920            tokio::time::sleep(Duration::from_millis(100)).await;
18921            guard.release();
18922        });
18923        let mut st = state(10);
18924        let budget = Duration::from_millis(50);
18925        let wait = repository_wait(&st, None);
18926        let (outcomes, _) = with_cache_lease(
18927            &mut st,
18928            Some(&cache),
18929            "gate",
18930            "gate",
18931            tmp.path(),
18932            "abc1234",
18933            budget,
18934            "test gate",
18935            &wait,
18936            |_, available| async move {
18937                assert_eq!(
18938                    available, budget,
18939                    "contention must not shorten verification"
18940                );
18941                (Vec::new(), false, Vec::new())
18942            },
18943        )
18944        .await
18945        .unwrap();
18946        release.await.unwrap();
18947        assert!(outcomes.is_empty(), "{outcomes:?}");
18948        assert!(
18949            st.events
18950                .iter()
18951                .any(|event| event.message.contains("waiting for"))
18952        );
18953        assert!(st.resource_wait.is_none());
18954        // The verification body no longer owns the lease after returning.
18955        hold(&cache).release();
18956    }
18957
18958    #[tokio::test]
18959    async fn verification_cache_wait_expires_without_starting_commands() {
18960        crate::run::pin_test_home();
18961        let tmp = tempfile::tempdir().unwrap();
18962        let cache = tmp.path().join("target");
18963        let _held = hold(&cache);
18964        let mut st = state(0);
18965        let wait = repository_wait(&st, None);
18966        let (outcomes, _) = with_cache_lease(
18967            &mut st,
18968            Some(&cache),
18969            "gate",
18970            "gate",
18971            tmp.path(),
18972            "abc1234",
18973            Duration::from_secs(60),
18974            "test gate",
18975            &wait,
18976            |_, _| async { panic!("commands must not start without the cache lease") },
18977        )
18978        .await
18979        .unwrap();
18980        assert_eq!(outcomes.len(), 1);
18981        assert!(outcomes[0].resource_blocked);
18982        assert!(st.resource_wait.is_none());
18983    }
18984
18985    #[tokio::test]
18986    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18987        crate::run::pin_test_home();
18988        let tmp = tempfile::tempdir().unwrap();
18989        let cache = tmp.path().join("target-a");
18990        let guard = hold(&cache);
18991        // Held for longer than the agent's whole 1s budget.
18992        tokio::spawn(async move {
18993            tokio::time::sleep(Duration::from_millis(1500)).await;
18994            guard.release();
18995        });
18996        let mut st = state(60);
18997        let results = run(
18998            vec![job(
18999                sh("alpha", "echo ok"),
19000                tmp.path(),
19001                Duration::from_secs(1),
19002            )],
19003            Arc::new(Semaphore::new(1)),
19004            Some(&cache),
19005            &mut st,
19006        )
19007        .await;
19008        assert!(
19009            matches!(results[0].2, AgentOutcome::Ok(_)),
19010            "the wait must not be charged to the agent"
19011        );
19012        assert!(st.resource_block.is_none());
19013        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19014        assert!(st.resource_wait.is_none() && st.active.is_empty());
19015    }
19016
19017    #[tokio::test]
19018    #[cfg(unix)]
19019    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19020        use std::os::fd::AsRawFd as _;
19021        crate::run::pin_test_home();
19022        let tmp = tempfile::tempdir().unwrap();
19023        crate::git::git(tmp.path(), &["init"]).await.unwrap();
19024        let common = tmp.path().join(".git");
19025        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19026        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19027        assert_eq!(
19028            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19029            0
19030        );
19031        let mut st = state(1);
19032        st.config.coordination = crate::config::test_coordination();
19033        let mut reader = job(
19034            sh("reader", "echo must-not-run"),
19035            tmp.path(),
19036            Duration::from_secs(30),
19037        );
19038        reader.allow_write = false;
19039        let results = run(
19040            vec![reader.clone()],
19041            Arc::new(Semaphore::new(1)),
19042            None,
19043            &mut st,
19044        )
19045        .await;
19046        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19047        assert_eq!(results[0].1.turns, 0);
19048        assert!(FailClass::of(&results[0].2).is_none());
19049        assert_eq!(
19050            st.resource_block.as_ref().unwrap().kind,
19051            "repository coordination"
19052        );
19053        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19054        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19055        drop(external);
19056        let editor = crate::repo_lock::SeatLease::for_checkout(
19057            tmp.path(),
19058            true,
19059            crate::config::test_coordination().as_ref(),
19060        )
19061        .await
19062        .unwrap();
19063        let pause = Pause::new();
19064        let interrupt = Pause::new();
19065        let prompts = Prompts::default();
19066        let ctx = WaveCtx {
19067            run: "r",
19068            node: "review",
19069            prompts: &prompts,
19070            cache: None,
19071            round: None,
19072            carry_seats: false,
19073            pauses: Some((&pause, &interrupt)),
19074        };
19075        let (results, ()) = tokio::join!(
19076            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19077            async {
19078                tokio::time::sleep(Duration::from_millis(50)).await;
19079                pause.park();
19080            }
19081        );
19082        assert!(
19083            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19084        );
19085        assert_eq!(results[0].1.turns, 0);
19086        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19087        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19088        drop(editor);
19089    }
19090
19091    #[tokio::test]
19092    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19093        crate::run::pin_test_home();
19094        let tmp = tempfile::tempdir().unwrap();
19095        let cache = tmp.path().join("target-b");
19096        let _held = hold(&cache);
19097        let mut st = state(1);
19098        let results = run(
19099            vec![job(
19100                sh("alpha", "echo ok"),
19101                tmp.path(),
19102                Duration::from_secs(1),
19103            )],
19104            Arc::new(Semaphore::new(1)),
19105            Some(&cache),
19106            &mut st,
19107        )
19108        .await;
19109        let (_, seat, out) = &results[0];
19110        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19111        assert!(wave_blocked(&results));
19112        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19113        assert_eq!(seat.turns, 0, "nothing was asked");
19114        assert!(
19115            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19116            "no agent was spawned"
19117        );
19118        assert!(FailClass::of(out).is_none(), "not a failure class");
19119        let block = st.resource_block.as_ref().expect("recorded for the resume");
19120        assert_eq!(block.kind, "shared build cache");
19121        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19122        assert!(st.active.is_empty());
19123    }
19124
19125    #[tokio::test]
19126    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19127        crate::run::pin_test_home();
19128        let tmp = tempfile::tempdir().unwrap();
19129        let mut st = state(1);
19130        let results = run(
19131            vec![job(
19132                sh("alpha", "echo ok"),
19133                tmp.path(),
19134                Duration::from_secs(1),
19135            )],
19136            Arc::new(Semaphore::new(0)),
19137            None,
19138            &mut st,
19139        )
19140        .await;
19141        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19142        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19143        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19144    }
19145
19146    #[tokio::test]
19147    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19148        crate::run::pin_test_home();
19149        let tmp = tempfile::tempdir().unwrap();
19150        let mut st = state(60);
19151        let results = run(
19152            vec![job(
19153                sh("alpha", "sleep 30"),
19154                tmp.path(),
19155                Duration::from_secs(1),
19156            )],
19157            Arc::new(Semaphore::new(1)),
19158            None,
19159            &mut st,
19160        )
19161        .await;
19162        let out = &results[0].2;
19163        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19164        let class = FailClass::of(out).expect("a failure");
19165        assert_eq!(class, FailClass::Timeout);
19166        assert!(should_hand_over(None, &class), "existing handover policy");
19167        assert!(st.resource_block.is_none());
19168    }
19169}