Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// How much verification output is kept and fed back to the fixer.
64const OUTPUT_TAIL: usize = 8_000;
65
66/// Bytes of a failing command's output kept in an event, so the reason a run
67/// stopped is readable from the report without opening `run.json`.
68const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
71/// command's pid before releasing the build cache's lease.
72const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
75/// command's pid to actually exit before giving up and releasing anyway.
76///
77/// A timeout means the process was asked to die (`kill_on_drop`,
78/// `start_kill`), not that it already has — on Windows in particular that can
79/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
80/// the instant the command returns would let the very next acquirer (this
81/// run's own next round, another run's verification, the janitor's prune)
82/// start touching the same directory while it might still be writing to it,
83/// so this polls the actual pid — real confirmation, not a fixed guess —
84/// until it is gone or this ceiling is reached. It is still not full
85/// process-tree reaping: a grandchild the timed-out process spawned and that
86/// outlives it independently is invisible to a pid check, and continuing to
87/// observe and collect *that* stays a different piece of work with its own
88/// owner. Set generously because the common case returns early the moment
89/// the pid is confirmed gone, not because every timeout pays this in full.
90const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92/// Consecutive review rounds with no tree progress (see
93/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
94/// instead of spending the rest of the round budget.
95///
96/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
97/// legitimately finds nothing left to change (its previous round's fix already
98/// covered it, and this round's reviewers re-raised only nits) looks the same
99/// as one that is spinning, for exactly one round. Two in a row is where the
100/// two stop being distinguishable, and a review round on this workload has
101/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
102/// third attempt at a tree that has not moved twice running is pure cost.
103/// This does not touch `review_rounds` itself, which stays the operator's
104/// call.
105pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
108/// a base that moved before giving up and leaving the run `Blocked` for a
109/// person.
110///
111/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
112/// that keeps moving faster than a run can catch it is not something more
113/// rebasing fixes, it is a person's call. Not the same *number as*
114/// `land_rounds` - this budget is spent before a pull request exists, land's
115/// after - but bounded for the identical reason, so it uses the same
116/// default. Counted across both call sites in [`Runner::finish_after_tally`]
117/// (once before review, once before the gate), because either one finding
118/// the base still moving is the same signal.
119const BASE_SYNC_ROUNDS: usize = 4;
120
121/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
122/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
123/// reply held no [`FixReport`].
124///
125/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
126/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
127/// "I'll pause here until the `cargo make check` background run reports
128/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
129/// No `FixReport` was ever collected from that seat, and the run moved on to
130/// the next review round regardless.
131///
132/// Bounded independently of `review_rounds` and `graph.retries`: this
133/// recovers one seat's missing report mid-round, not a new round of review or
134/// an ordinary parse retry, and must not itself become the unbounded wait the
135/// rest of this module exists to avoid.
136const MAX_FIX_CONTINUATIONS: usize = 2;
137
138/// One queued agent invocation.
139///
140/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
141/// CLI hung up on its own stream is asked again from the same job rather than
142/// rebuilt from scratch. See [`Runner::resume_undelivered`].
143#[derive(Clone)]
144struct SeatJob {
145    spec: AgentSpec,
146    seat: SeatState,
147    cwd: PathBuf,
148    prompt: String,
149    timeout: Duration,
150    allow_write: bool,
151    sessions: bool,
152    artifacts: PathBuf,
153    stem: String,
154    /// The prompt for a seat that has been handed to another roster agent
155    /// (a fresh session): everything the original seat would have
156    /// remembered. `None` when `prompt` already carries it, as the first
157    /// ranking and the implement prompt do. Never a resume-style prompt.
158    handover: Option<String>,
159}
160
161/// How the graph reads one agent invocation.
162///
163/// Quota is split out from an ordinary failure on purpose: a rate-limited call
164/// is known to fail again if retried now, so the retry loop must not spend an
165/// attempt on it. `Dropped` is split out for the opposite reason: unlike
166/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
167/// error JSON, never the agent's answer — a caller that matched only
168/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
169/// left to read that JSON as if it were usable output. `resume_undelivered`
170/// is the only caller that acts on it; everywhere else it is reported like an
171/// ordinary failure.
172enum AgentOutcome {
173    /// A usable output.
174    Ok(AgentOutput),
175    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
176    Quota(AgentOutput),
177    /// The CLI hung up on its own stream after billed work. See
178    /// [`agent::AgentOutput::work_undelivered`].
179    Dropped(AgentOutput),
180    /// Any other failure: a timeout, a bad exit code, an empty reply.
181    Failed(String),
182    /// A resource prevented admission or interrupted execution. Not an agent
183    /// failure: never advance a fallback chain or spend an attempt on it.
184    /// Post-spawn interruptions carry a typed error into wave, which records
185    /// implementation preservation before returning this outcome. This outcome
186    /// alone is not proof that no source changes were made.
187    /// [`RunState::resource_block`] says why.
188    ResourceBlocked(String),
189}
190
191/// A request to park the run at its next node boundary.
192///
193/// Cloning is how the request travels: the loop keeps one handle and hands a
194/// clone to each [`Runner`], and every clone points at the same flag. There
195/// is no channel because there is nothing to send - the only message is
196/// "park", it is idempotent, and a flag cannot be missed by a receiver that
197/// was not listening yet.
198///
199/// The boundary is what makes this cheap. Every node writes the run's state
200/// before the next one starts, and every node skips what is already recorded:
201/// `prep` returns early once candidates exist, `implement` asks only the seats
202/// with nothing on disk, `judge` returns early once judgements exist. So a
203/// parked run resumes into exactly the node it stopped before, and no agent
204/// work is thrown away. Killing the process mid-node, by contrast, loses
205/// whatever the seats in flight had not yet written - which for an implement
206/// wave is an hour of paid work.
207///
208/// A [`Runner`] watches two independent handles of this type - see
209/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
210/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
211/// clone covering the whole daemon's lifetime and is never asked to un-park,
212/// which is correct exactly because nothing is dispatched after it fires.
213/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
214/// that parks for an interrupted task must go on to run other tasks
215/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
216/// reusing the daemon-wide one.
217#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221    /// A pause nobody has asked for yet.
222    #[must_use]
223    pub fn new() -> Self {
224        Self::default()
225    }
226
227    /// Ask the run to park at its next node boundary. Idempotent.
228    pub fn park(&self) {
229        self.0.store(true, Ordering::SeqCst);
230    }
231
232    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
233    /// fold into the run's own `park` event - so an operator reading the run
234    /// later knows this was a deliberate interrupt rather than a shutdown or
235    /// a binary swap. The first reason recorded wins; a park already in
236    /// flight is not relabelled by a second, unrelated request.
237    pub fn park_because(&self, reason: impl Into<String>) {
238        self.park_for(ParkCause::Interrupt {
239            reason: reason.into(),
240        });
241    }
242
243    /// [`Pause::park`], recording the structured cause. The first cause
244    /// recorded wins, like [`Pause::park_because`].
245    pub fn park_for(&self, cause: ParkCause) {
246        let mut reason_guard = self
247            .1
248            .lock()
249            .unwrap_or_else(std::sync::PoisonError::into_inner);
250        if reason_guard.is_none() {
251            *reason_guard = Some(cause);
252        }
253        drop(reason_guard);
254        self.park();
255    }
256
257    /// Has a park been asked for?
258    #[must_use]
259    pub fn parked(&self) -> bool {
260        self.0.load(Ordering::SeqCst)
261    }
262
263    /// Why the park was asked for, when the caller used [`Pause::park_because`].
264    #[must_use]
265    pub fn reason(&self) -> Option<String> {
266        self.cause().map(|c| c.label())
267    }
268
269    /// The structured cause, when one was recorded.
270    #[must_use]
271    pub fn cause(&self) -> Option<ParkCause> {
272        self.1
273            .lock()
274            .unwrap_or_else(std::sync::PoisonError::into_inner)
275            .clone()
276    }
277}
278
279/// Drives one run.
280pub struct Runner {
281    /// Run state; public so the CLI can report on it.
282    pub state: RunState,
283    roles: ResolvedRoles,
284    sem: Arc<Semaphore>,
285    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
286    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
287    /// this is never the same handle as `interrupt`.
288    pause: Pause,
289    /// Set when `magi serve`'s interrupt scheduler wants this specific run
290    /// parked at its next node boundary, to let a task marked
291    /// [`crate::queue::Task::interrupt`] run alone before this one carries
292    /// on. Unlike `pause`, a fresh, unshared handle per run - see
293    /// [`Runner::watch_interrupt`].
294    interrupt: Pause,
295}
296
297/// Where the branch's own commits start: its merge base with the base branch
298/// as the remote has it now, else with the recorded `base_commit`. A branch
299/// rebased onto a base that moved past `base_commit` would otherwise count
300/// the base's commits as its own under `base_commit..branch`.
301async fn review_base(
302    repo: &Path,
303    remote: &str,
304    base_branch: &str,
305    base_commit: &str,
306    branch: &str,
307) -> String {
308    review_base_checked(repo, remote, base_branch, base_commit, branch)
309        .await
310        .0
311}
312
313/// [`review_base`] plus whether the base was read from a freshly fetched
314/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
315/// never older than `base_commit`'s view of the base), but the answer is then
316/// not trusted to rewrite a pull request's title.
317async fn review_base_checked(
318    repo: &Path,
319    remote: &str,
320    base_branch: &str,
321    base_commit: &str,
322    branch: &str,
323) -> (String, bool) {
324    let tracking = format!("{remote}/{base_branch}");
325    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326    if git::rev_exists(repo, &tracking).await
327        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328        && !mb.is_empty()
329    {
330        return (mb, fresh);
331    }
332    let mb = git::merge_base(repo, base_commit, branch)
333        .await
334        .ok()
335        .filter(|mb| !mb.is_empty())
336        .unwrap_or_else(|| base_commit.to_owned());
337    (mb, false)
338}
339
340/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
341/// when git cannot say or finds nothing: a stale list is better than a wrong
342/// or empty one.
343pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344    if !is_review_run(state) {
345        return;
346    }
347    let base = review_base(
348        &state.repo,
349        &state.config.merge.remote,
350        &state.base_branch,
351        &state.base_commit,
352        branch,
353    )
354    .await;
355    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356        && !subjects.is_empty()
357        && state.reviewed_commits.as_ref() != Some(&subjects)
358    {
359        state.reviewed_commits = Some(subjects);
360        state.save().ok();
361    }
362}
363
364/// Subjects of the base's commits between the recorded start and the branch's
365/// merge base: what a stale `base_commit..branch` would have mistaken for the
366/// branch's own work.
367async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368    let (base, trusted) = review_base_checked(
369        &state.repo,
370        &state.config.merge.remote,
371        &state.base_branch,
372        &state.base_commit,
373        branch,
374    )
375    .await;
376    if !trusted {
377        return None;
378    }
379    git::subjects(&state.repo, &state.base_commit, &base)
380        .await
381        .ok()
382}
383
384/// May an adopted pull request's title be replaced with `computed`? Only when
385/// it is empty, magi's own shape, or a base commit's subject that leaked in;
386/// a title a person wrote stays. Never when `computed` is itself a leak.
387pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389    if is_leak(computed) {
390        return false;
391    }
392    let cur = current.trim();
393    cur.is_empty()
394        || cur.starts_with(REVIEW_PROMPT_OPENING)
395        || cur.starts_with("chore: land candidate")
396        || cur.starts_with("magi: candidate")
397        || is_leak(cur)
398}
399
400/// The commit a run branches from: the base branch as the remote has it.
401///
402/// Two failures this replaces. A run used to branch off `HEAD` and so refused
403/// to start on a dirty tree, which made `magi serve` decline every task for as
404/// long as the operator had work in progress - most of the time. Branching off
405/// the *local* base branch fixed that and introduced a worse one: `land` merges
406/// the winner on GitHub, nothing updates the local ref, and the next run
407/// branches off a base missing everything the previous runs landed. Two tasks
408/// in a row from a phone would have had the second silently re-implementing
409/// against stale code and opening a pull request that reverted the first.
410///
411/// Only refs move here - no checkout, no local branch, no merge - so it is safe
412/// with uncommitted work in the tree. A machine with no network still starts:
413/// the fetch may fail and the local tip is used with a warning, because
414/// refusing to run offline is a worse failure than running against a base the
415/// operator can see for themselves.
416///
417/// One function, called by both entry points. Two answers to "where does a run
418/// branch from" is the kind of drift nobody notices until a diff is wrong.
419/// Bring the local `branch` in line with `<remote>/<branch>` before a review
420/// checks it out.
421///
422/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
423/// anything other than plain `git push` from this checkout (a jj colocated
424/// workspace, another clone) moves only the remote-tracking ref - so the local
425/// one can be a stale placeholder. It moves only when local is behind the remote or is an
426/// empty placeholder that diverged from it; unpushed local work is kept, and a real
427/// divergence is refused rather than guessed at.
428async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429    let tracking = format!("{remote}/{branch}");
430    let fetched = git::fetch(repo, remote, branch).await;
431    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432    let local_exists = git::branch_exists(repo, branch).await?;
433    if !fresh {
434        if !local_exists {
435            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436        }
437        tracing::warn!(
438            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439        );
440        return Ok(());
441    }
442    let remote_sha = git::rev_parse(repo, &tracking).await?;
443    if !local_exists {
444        git::git(repo, &["branch", branch, &tracking]).await?;
445        return Ok(());
446    }
447    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449        return Ok(());
450    }
451    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452        // Diverged. `reconcile` settles it only when it can prove nothing is
453        // lost: a local tip that is the remote's change rebased is pushed over
454        // it (lease pinned to the tip read here), a tip whose every extra
455        // commit is empty is a placeholder the remote's work replaced, and
456        // anything else is two different changes - a question for a person.
457        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458            .await?
459        {
460            crate::reconcile::Reconciliation::Pushed => {
461                tracing::warn!(
462                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463                    short(&local_sha),
464                    short(&remote_sha)
465                );
466                return Ok(());
467            }
468            crate::reconcile::Reconciliation::Placeholder => {}
469            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470        }
471    }
472    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473    if !out.ok() {
474        bail!(
475            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476            short(&local_sha),
477            short(&remote_sha),
478            out.stderr
479        );
480    }
481    tracing::warn!(
482        "local `{branch}` was stale: fast-forwarded {} -> {}",
483        short(&local_sha),
484        short(&remote_sha)
485    );
486    Ok(())
487}
488
489async fn resolve_base(
490    repo: &Path,
491    base_branch: &str,
492    remote: &str,
493    coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495    let _lease = crate::repo_lock::SeatLease::operation_with(
496        repo,
497        coordination,
498        &crate::repo_lock::WaitPolicy::current(),
499    )
500    .await?;
501    let tracking = format!("{remote}/{base_branch}");
502    let fetched = git::fetch(repo, remote, base_branch).await;
503    if let Ok(out) = &fetched
504        && out.ok()
505        && git::rev_exists(repo, &tracking).await
506    {
507        return Ok(crate::run::FetchedBase {
508            remote: remote.to_owned(),
509            branch: base_branch.to_owned(),
510            commit: git::rev_parse(repo, &tracking).await?,
511            fetched_at: jiff::Timestamp::now(),
512        });
513    }
514    let why = match &fetched {
515        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516        Ok(_) => format!("{remote} has no {base_branch}"),
517        Err(e) => e.to_string(),
518    };
519    // No fallback to the local branch: it may be behind, and branching off an
520    // old commit is the stale-checkout bug this check exists to prevent.
521    bail!(
522        "cannot read {tracking} ({why}); refusing to branch off the local \
523         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524         base / remote in magi.toml. This is launcher preparation: check the \
525         launcher's Git administration access, network and repository coordination \
526         before retrying; an unattended seat cannot approve those permissions"
527    )
528}
529
530/// Outcome of the shared fixer chain, before caller-owned stop policy.
531enum FixStep {
532    Resource,
533    Recorded { report_lost: bool },
534}
535
536/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
537/// on an early return or a panic.
538///
539/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
540/// manual `magi fix` invocations against the same run are otherwise
541/// invisible to each other and would race to remove and recreate the same
542/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
543/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
544/// which is only ever reclaimed later, out of band, by
545/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
546/// `magi fix` invocation is not necessarily running under either of those, so
547/// nothing would ever sweep a lock a killed or crashed process left behind.
548/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
549/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
550/// lease use: an unreadable or unparsable pid, or a liveness query the
551/// platform cannot answer, reads as alive and the lock is left in place.
552struct FixClaim {
553    path: PathBuf,
554}
555
556impl FixClaim {
557    fn acquire(dir: &Path) -> Result<Self> {
558        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559        let path = dir.join("fix.lock");
560        let retry = crate::lock_file::Retry::new(None);
561        match Self::create(&path, retry) {
562            Ok(claim) => Ok(claim),
563            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564                if Self::reclaim_if_dead(&path) {
565                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566                } else {
567                    bail!(
568                        "another repair or `magi fix` is already running for this run ({} exists)",
569                        path.display()
570                    )
571                }
572            }
573            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574        }
575    }
576
577    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578        let mut f = retry.create(path)?;
579        use std::io::Write as _;
580        // Read back by `reclaim_if_dead` on a later, stuck invocation.
581        writeln!(f, "{}", std::process::id())?;
582        Ok(Self {
583            path: path.to_owned(),
584        })
585    }
586
587    /// True if the lock named a process confirmed dead, in which case it was
588    /// also removed. Never true on an unreadable file, an unparsable pid, or
589    /// a liveness query the platform cannot answer — see this type's own doc.
590    fn reclaim_if_dead(path: &Path) -> bool {
591        let dead = std::fs::read_to_string(path)
592            .ok()
593            .and_then(|body| body.trim().parse::<u32>().ok())
594            .is_some_and(|pid| !crate::proc::pid_alive(pid));
595        dead && std::fs::remove_file(path).is_ok()
596    }
597}
598
599impl Drop for FixClaim {
600    fn drop(&mut self) {
601        let _ = std::fs::remove_file(&self.path);
602    }
603}
604
605/// Holds the mutable runner as well as its durable lease: cancellation must
606/// clear both copies of ownership, including state a caller may save later.
607struct DriverExecution<'a> {
608    runner: &'a mut Runner,
609    driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613    fn drop(&mut self) {
614        self.driver.release_local(&mut self.runner.state);
615    }
616}
617
618impl Runner {
619    /// Start a fresh run against `repo`.
620    pub async fn start(
621        repo: &Path,
622        instruction: String,
623        config: Config,
624        origin: Origin,
625    ) -> Result<Self> {
626        Self::start_naming(repo, instruction, "", config, origin).await
627    }
628
629    /// [`Runner::start`] for a queued task. Titles and instructions never
630    /// select implementation sources. The title argument is retained for callers.
631    pub async fn start_naming(
632        repo: &Path,
633        instruction: String,
634        _also_scan: &str,
635        config: Config,
636        origin: Origin,
637    ) -> Result<Self> {
638        let repo = git::toplevel(repo).await?;
639        let missing = agent::missing_programs(&config.agents);
640        if !missing.is_empty() {
641            bail!(
642                "these agent programs are not on PATH: {}. Fix the roster in \
643                 magi.toml or install them.",
644                missing.join(", ")
645            );
646        }
647        let base_branch =
648            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649                .await?;
650        let fetched_base = resolve_base(
651            &repo,
652            &base_branch,
653            &config.merge.remote,
654            config.coordination.as_ref(),
655        )
656        .await?;
657        let base_commit = fetched_base.commit.clone();
658        let roles = config.resolve_roles()?;
659        let max_parallel = config.graph.max_parallel.max(1);
660        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661        // Recorded before the first save, so a crash right after minting
662        // cannot leave a run with no origin. Legibility only: nothing reads it
663        // to decide anything.
664        state.fetched_base = Some(fetched_base);
665        state.origin = Some(origin);
666        state.reserve()?;
667        state.event("start", format!("run {} created", state.id));
668        state.save()?;
669        Ok(Self {
670            state,
671            roles,
672            sem: Arc::new(Semaphore::new(max_parallel)),
673            pause: Pause::new(),
674            interrupt: Pause::new(),
675        })
676    }
677
678    /// Open a review-only run against work that already exists on `branch`.
679    ///
680    /// The expensive half of the graph is the implement wave — measured at
681    /// 111 and 134 internal tool-loop turns on this repository, against a
682    /// handful for a judge or a reviewer. The cheap half is worth running on
683    /// hand-written work too, and there was no way to reach it.
684    ///
685    /// No new state and no schema change are needed: a run with **one** viable
686    /// candidate and a tally already decided degrades `execute` to exactly
687    /// review → gate → merge, because `judge` skips a single-candidate field,
688    /// `deliberate` has fewer than two first choices to reconcile, `vote`
689    /// returns early, `tally` is already present and `fold_losers` has no
690    /// losers. Resuming such a run therefore does the right thing as well.
691    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692        Self::review_taking_over(repo, branch, config, None, origin).await
693    }
694
695    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
696    /// at the same task still has `branch` checked out, its worktree is
697    /// released first if that is safe (see [`crate::handover`]), and the
698    /// review refuses with the reason if it is not. `None` is a hand-run
699    /// review: it has no earlier attempts, so only a worktree of a dead run
700    /// magi recorded itself can be released.
701    pub async fn review_taking_over(
702        repo: &Path,
703        branch: &str,
704        config: Config,
705        takeover: Option<crate::handover::Takeover>,
706        origin: Origin,
707    ) -> Result<Self> {
708        let repo = git::toplevel(repo).await?;
709        let missing = agent::missing_programs(&config.agents);
710        if !missing.is_empty() {
711            bail!(
712                "these agent programs are not on PATH: {}. Fix the roster in \
713                 magi.toml or install them.",
714                missing.join(", ")
715            );
716        }
717        let base_branch =
718            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719                .await?;
720        if base_branch == branch {
721            bail!("`{branch}` is the base branch; there is nothing to review against");
722        }
723        let fetched_base = resolve_base(
724            &repo,
725            &base_branch,
726            &config.merge.remote,
727            config.coordination.as_ref(),
728        )
729        .await?;
730        let base_commit = fetched_base.commit.clone();
731
732        let roles = config.resolve_roles()?;
733        let max_parallel = config.graph.max_parallel.max(1);
734        let mut state = RunState::new(
735            repo.clone(),
736            base_branch,
737            base_commit.clone(),
738            String::new(),
739            config,
740        );
741        state.fetched_base = Some(fetched_base);
742        state.origin = Some(origin);
743        state.reserve()?;
744
745        // Released before anything else touches the branch: a stale local
746        // branch is moved with `git branch -f`, which git refuses while an
747        // earlier attempt's worktree still has it checked out. Everything
748        // after this point that can fail puts the old run back.
749        // A hand-run review has no task, hence no earlier attempts, but a
750        // worktree of a dead run magi made may still be released.
751        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752            earlier: Vec::new(),
753            home: crate::run::home(),
754            choice: None,
755            task_instruction: None,
756        });
757        let wait = repository_wait(&state, None);
758        let coordination_config = state.config.coordination.clone();
759        let coordination =
760            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761                .await?;
762        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763        if let Some(released) = &released {
764            state.event(
765                "release",
766                format!(
767                    "took `{branch}` over from run {}: its worktree was released: {}",
768                    crate::run::short_of(&released.old_id),
769                    released.audit
770                ),
771            );
772        }
773        // The owner's answer to an earlier divergence question is applied
774        // here: after the release (git will not move a checked-out branch)
775        // and before the sync that would otherwise ask again.
776        if let Some(choice) = takeover.choice.as_ref()
777            && let Err(e) =
778                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779                    .await
780        {
781            if let Some(released) = &released {
782                released.restore(&repo, branch).await;
783            }
784            return Err(e.context("applying the owner's answer about the diverged branch"));
785        }
786        // An earlier attempt of this task that published the task head, and
787        // whose work is the branch under review, hands that head over: the
788        // review run updates the same pull request. Without one (a legacy
789        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
790        // own head and its own open pull request is adopted as before.
791        for id in takeover.earlier.iter().rev() {
792            if let Ok(old) = RunState::load_under(id, &takeover.home)
793                && let Some(head) = old.pr_head.clone()
794                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795            {
796                state.pr_head = Some(head);
797                break;
798            }
799        }
800        drop(coordination);
801        let opened = Self::open_review(
802            &repo,
803            branch,
804            state,
805            roles,
806            max_parallel,
807            base_commit,
808            &takeover,
809        )
810        .await;
811        if opened.is_err()
812            && let Some(released) = &released
813        {
814            let _coordination = crate::repo_lock::SeatLease::operation_with(
815                &repo,
816                coordination_config.as_ref(),
817                &wait,
818            )
819            .await?;
820            released.restore(&repo, branch).await;
821        }
822        opened
823    }
824
825    /// Recover exactly the operator-selected public implementation, without touching old sources.
826    pub async fn recover_pr(
827        repo: &Path,
828        task: &str,
829        selection: crate::pr_recovery::Selection,
830        mut config: Config,
831    ) -> Result<Self> {
832        selection.configure(&mut config)?;
833        selection.check_task(task, &selection.instruction)?;
834        let base_branch =
835            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836                .await?;
837        let fetched_base = resolve_base(
838            repo,
839            &base_branch,
840            &config.merge.remote,
841            config.coordination.as_ref(),
842        )
843        .await?;
844        let base_commit = fetched_base.commit.clone();
845        selection
846            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847            .await?;
848        Box::pin(selection.verify_source(repo)).await?;
849        if let Some(proof) = &selection.source_proof {
850            anyhow::ensure!(
851                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852                "source rebase base is outside the fetched landing base"
853            );
854        }
855        let roles = config.resolve_roles()?;
856        let max_parallel = config.graph.max_parallel.max(1);
857        let mut state = RunState::new(
858            repo.to_path_buf(),
859            base_branch,
860            base_commit.clone(),
861            selection.instruction.clone(),
862            config,
863        );
864        state.fetched_base = Some(fetched_base);
865        state.origin = Some(crate::run::Origin::queue(task));
866        state.pr_head = Some(selection.branch.clone());
867        state.pr_recovery = Some(selection.clone());
868        state.reserve()?;
869        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870            // Never leave a reserved record behind as a phantom owner of the head.
871            state.discard_reserved();
872            return Err(e);
873        }
874        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875    }
876
877    async fn open_selected(
878        repo: &Path,
879        state: RunState,
880        roles: ResolvedRoles,
881        max_parallel: usize,
882        base_commit: String,
883        selection: crate::pr_recovery::Selection,
884    ) -> Result<Self> {
885        let mut state = state;
886        let branch = state.branch_for('A');
887        let wait = repository_wait(&state, None);
888        let prepared = async {
889            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890            git::git(repo, &["branch", &branch, selection.source()]).await?;
891            drop(coordination);
892            anyhow::Ok(())
893        }
894        .await;
895        if let Err(e) = prepared {
896            // Nothing was exposed yet: do not leave a reserved record as a
897            // phantom owner of the public head.
898            state.discard_reserved();
899            return Err(e);
900        }
901        state.event(
902            "recovery",
903            format!(
904                "selected implementation {} for {} (public lease {})",
905                selection.source(),
906                selection.url,
907                selection.head
908            ),
909        );
910        let takeover = crate::handover::Takeover {
911            earlier: Vec::new(),
912            home: crate::run::home(),
913            choice: None,
914            task_instruction: Some(selection.instruction.clone()),
915        };
916        if let Err(e) = state.save() {
917            state.discard_reserved();
918            return Err(e);
919        }
920        let id = state.id.clone();
921        let opened = Self::open_review(
922            repo,
923            &branch,
924            state,
925            roles,
926            max_parallel,
927            base_commit,
928            &takeover,
929        )
930        .await;
931        match opened {
932            Ok(runner) => Ok(runner),
933            Err(e) => {
934                if let Ok(mut partial) = RunState::load(&id) {
935                    partial.status = RunStatus::Blocked;
936                    partial.driver_exited = true;
937                    partial.event(
938                        "recovery",
939                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
940                    );
941                    partial.save()?;
942                }
943                Err(crate::pr_recovery::StartupFailed {
944                    run: id,
945                    cause: format!("{e:#}"),
946                }
947                .into())
948            }
949        }
950    }
951
952    /// What the newest earlier attempt that reviewed `branch` leaves for this
953    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
954    /// no earlier attempts. Reads the earlier record only: ownership of its
955    /// worktree and branch is [`crate::handover`]'s business.
956    async fn inherit(
957        repo: &Path,
958        branch: &str,
959        base_commit: &str,
960        takeover: &crate::handover::Takeover,
961    ) -> Option<crate::recovery::RecoveryRecord> {
962        use crate::recovery::{Relation, assess};
963        let newest = takeover.earlier.last()?;
964        let mut chosen = None;
965        for id in takeover.earlier.iter().rev() {
966            if let Ok(old) = RunState::load_under(id, &takeover.home)
967                && !old.reviews.is_empty()
968                && (old.winner().is_some_and(|w| w.branch == branch)
969                    || old.pr_head.as_deref() == Some(branch))
970            {
971                chosen = Some(old);
972                break;
973            }
974        }
975        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976            .await
977            .unwrap_or_default();
978        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979            Some(last) if last.head == tip => Relation::Same,
980            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981            _ => Relation::Diverged,
982        };
983        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984        Some(assess(from, chosen.as_ref(), base_commit, relation))
985    }
986
987    /// The half of [`Runner::review_taking_over`] that can fail after an
988    /// earlier attempt's worktree was released.
989    async fn open_review(
990        repo: &Path,
991        branch: &str,
992        mut state: RunState,
993        roles: ResolvedRoles,
994        max_parallel: usize,
995        base_commit: String,
996        takeover: &crate::handover::Takeover,
997    ) -> Result<Self> {
998        let wait = repository_wait(&state, None);
999        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000        if state.pr_recovery.is_none() {
1001            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002        }
1003        // The commit subjects are the closest thing to a task statement that
1004        // existing work carries, and the reviewers are told as much.
1005        let start = review_base(
1006            repo,
1007            &state.config.merge.remote,
1008            &state.base_branch,
1009            &base_commit,
1010            branch,
1011        )
1012        .await;
1013        let log = git::log_oneline(repo, &start, branch)
1014            .await
1015            .unwrap_or_default();
1016        let log_text = if log.trim().is_empty() {
1017            "(no commit messages)"
1018        } else {
1019            log.trim()
1020        };
1021        let mut instruction = format!(
1022            "Review the work already on branch `{branch}`. There is no task \
1023             statement: what the change claims to do is whatever its commits \
1024             say.\n\n{log_text}"
1025        );
1026        // A retry of a task whose earlier attempt left reviews behind carries
1027        // that attempt's evidence, and the task's own statement with it: the
1028        // completion criteria are the task's, not whatever the commits claim.
1029        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030            if let Some(task) = takeover
1031                .task_instruction
1032                .as_deref()
1033                .filter(|t| !t.trim().is_empty())
1034                && rec.reviewed_head.is_some()
1035            {
1036                instruction = format!(
1037                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038                    task.trim()
1039                );
1040                rec.criteria_inherited = true;
1041            }
1042            state.event("recovery", rec.summary());
1043            state.recovery = Some(rec);
1044        }
1045        if let Some(selected) = &state.pr_recovery {
1046            instruction = selected.instruction.clone();
1047        }
1048        state.task_instruction = takeover.task_instruction.clone();
1049        state.instruction = instruction;
1050        state.reviewed_commits = Some(
1051            git::subjects(repo, &start, branch)
1052                .await
1053                .unwrap_or_default(),
1054        );
1055
1056        // A takeover must not forget a hold that a previous review observed,
1057        // even if repairs have since removed it from the branch.
1058        for id in &takeover.earlier {
1059            if let Ok(old) = RunState::load_under(id, &takeover.home)
1060                && old.repo == state.repo
1061                && (old.pr_head.as_deref() == Some(branch)
1062                    || old.winner().is_some_and(|w| w.branch == branch))
1063            {
1064                for hold in old.branch_owner_holds {
1065                    if !state.branch_owner_holds.contains(&hold) {
1066                        state.event(
1067                            "approval-policy",
1068                            format!(
1069                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070                                hold.control.key(),
1071                                hold.file,
1072                                hold.head
1073                            ),
1074                        );
1075                        state.branch_owner_holds.push(hold);
1076                    }
1077                }
1078            }
1079        }
1080        let reviewed_head = git::rev_parse(repo, branch).await?;
1081        state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083        // An attached worktree, so the fixer's commits land on the branch under
1084        // review rather than on a detached head nobody will look at again.
1085        let worktree = state.worktree_root().join("under-review");
1086        if let Some(parent) = worktree.parent() {
1087            tokio::fs::create_dir_all(parent).await.ok();
1088        }
1089        let path = worktree.to_string_lossy().to_string();
1090        git::git(repo, &["worktree", "add", &path, branch])
1091            .await
1092            .with_context(|| {
1093                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094            })?;
1095
1096        let setup = match ensure_setup_config(&mut state, repo).await {
1097            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098            Err(e) => Err(e),
1099        };
1100        if let Err(e) = setup {
1101            git::worktree_remove_clean(repo, &worktree).await.ok();
1102            return Err(e);
1103        }
1104        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105            .await
1106            .unwrap_or(0);
1107        if commits == 0 {
1108            git::worktree_remove_clean(repo, &worktree).await.ok();
1109            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110        }
1111        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112            .await
1113            .map(|f| f.len())
1114            .unwrap_or(0);
1115        if files == 0
1116            && let (Ok(head_tree), Ok(base_tree)) = (
1117                git::tree_of(&worktree, "HEAD").await,
1118                git::tree_of(&worktree, &base_commit).await,
1119            )
1120            && head_tree == base_tree
1121        {
1122            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123            git::worktree_remove_clean(repo, &worktree).await.ok();
1124            bail!(
1125                "`{branch}` at {} has a tree identical to base {}; this usually means \
1126                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127                 against `{}/{branch}`) rather than an empty change",
1128                short(&head),
1129                short(&base_commit),
1130                state.config.merge.remote
1131            );
1132        }
1133        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134            .await
1135            .unwrap_or_default();
1136
1137        state.candidates.push(Candidate {
1138            index: 0,
1139            label: 'A',
1140            // Not an agent id on purpose: nothing in the roster wrote this, and
1141            // the stats tables must not credit anyone with a win for it.
1142            agent: EXISTING_BRANCH.to_owned(),
1143            branch: branch.to_owned(),
1144            worktree,
1145            summary: String::new(),
1146            stat,
1147            files,
1148            commits,
1149            empty: false,
1150            failed: None,
1151            verified_noop: None,
1152            duration_ms: 0,
1153            folded: false,
1154        });
1155        state.tally = Some(Tally {
1156            first_choice: BTreeMap::from([('A', 0)]),
1157            borda: BTreeMap::new(),
1158            winner: 'A',
1159            rankings: 0,
1160            unanimous_initial: false,
1161            deliberated: false,
1162            changed_votes: 0,
1163            unanimous_final: false,
1164            tie_break: None,
1165            // No panel sat, so no quorum applies. Zero judges is the correct
1166            // number for work that never competed, and must not be reported as
1167            // a collapsed panel.
1168            judges: 0,
1169            present: 0,
1170            quorum: 0,
1171            met_quorum: true,
1172            uncontested: Some("review-only run: nothing competed".to_owned()),
1173        });
1174        state.status = RunStatus::Reviewing;
1175        state.prep_complete = Some(true);
1176        state.event(
1177            "start",
1178            format!(
1179                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180                state.id
1181            ),
1182        );
1183        state.save()?;
1184        Ok(Self {
1185            state,
1186            roles,
1187            sem: Arc::new(Semaphore::new(max_parallel)),
1188            pause: Pause::new(),
1189            interrupt: Pause::new(),
1190        })
1191    }
1192
1193    /// Reopen an existing run.
1194    pub fn resume(id: &str) -> Result<Self> {
1195        let state = RunState::load(id)?;
1196        crate::pr_head::ensure_owner(&state)?;
1197        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198            bail!(
1199                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200            );
1201        }
1202        if let Some(to) = &state.released_to {
1203            bail!(
1204                "run {} cannot be resumed: its worktree was released to run {}",
1205                state.short(),
1206                crate::run::short_of(to)
1207            );
1208        }
1209        let roles = state.config.resolve_roles()?;
1210        if let Some(id) = &state.config.graph.selected_implementer
1211            && state.candidates.iter().any(|c| c.agent != *id)
1212        {
1213            bail!(
1214                "replacement implementer pin does not match the saved candidates; task must stay held"
1215            );
1216        }
1217        let max_parallel = state.config.graph.max_parallel.max(1);
1218        Ok(Self {
1219            state,
1220            roles,
1221            sem: Arc::new(Semaphore::new(max_parallel)),
1222            pause: Pause::new(),
1223            interrupt: Pause::new(),
1224        })
1225    }
1226
1227    /// Walk the graph to a terminal state, skipping nodes already recorded.
1228    ///
1229    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1230    /// the phone - ends here, so this is the one place a run that ended
1231    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1232    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1233    pub async fn execute(&mut self) -> Result<()> {
1234        if let Some(selection) = &self.state.pr_recovery
1235            && (selection.source_proof.is_some() || selection.source_run.is_some())
1236        {
1237            Box::pin(selection.verify_source(&self.state.repo)).await?;
1238        }
1239        // Queue claims cover daemon work; this also serializes manual drivers
1240        // consuming a repair answer, including the new review and decision.
1241        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242            || self.state.land_approval.is_some()
1243            || self.state.merge_decision.is_some()
1244            || !self.state.merge_repairs.is_empty()
1245            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246        {
1247            let claim = FixClaim::acquire(&self.state.dir())?;
1248            let disk = RunState::load(&self.state.id)?;
1249            let checkpoint = |s: &RunState| {
1250                (
1251                    s.merge_repairs.len(),
1252                    s.merge_repairs
1253                        .last()
1254                        .map(|r| (r.question.clone(), r.phase)),
1255                    s.land_approval
1256                        .as_ref()
1257                        .map(|a| (a.question.clone(), a.head.clone())),
1258                )
1259            };
1260            if checkpoint(&disk) != checkpoint(&self.state) {
1261                bail!(
1262                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263                );
1264            }
1265            Some(claim)
1266        } else {
1267            None
1268        };
1269        let mut execution = self.claim_driver().await?;
1270        let runner = &mut *execution.runner;
1271        let wait = runner.repository_wait();
1272        // Keep the graph's large future off the callers' stack, including
1273        // standalone task attempts running on a default-sized test thread.
1274        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275        let result = match result {
1276            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277                record_repository_block(&mut runner.state, &e);
1278                runner.park_here()?;
1279                Ok(())
1280            }
1281            other => other,
1282        };
1283        execution.driver.finish(&mut runner.state).await?;
1284        let ended = if result.is_err() {
1285            Some({
1286                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1287                if let Err(error) = &result {
1288                    notice.message = format!(
1289                        "Run {} stopped with an error: {}",
1290                        runner.state.id,
1291                        error.to_string().lines().next().unwrap_or("unknown error")
1292                    );
1293                }
1294                notice
1295            })
1296        } else {
1297            crate::notices::run_ended(&runner.state)
1298        };
1299        if let Some(notice) = ended {
1300            crate::notices::raise(notice);
1301        }
1302        result
1303    }
1304
1305    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1306        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1307    }
1308
1309    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1310        let wait = self.repository_wait();
1311        coordinate_operation(&mut self.state, cwd, &wait).await
1312    }
1313
1314    /// Validate ownership and persist the driver claim in one critical section.
1315    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1316        let owner = match crate::pr_head::owner_lock_wait(
1317            &crate::run::home(),
1318            &self.state.id,
1319            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1320        )
1321        .await
1322        {
1323            Ok(owner) => owner,
1324            Err(error) => {
1325                let detail = format!("worktree maintenance or ownership update: {error:#}");
1326                // Admission did not own run.json: retain this resource outcome
1327                // in memory for queue settlement without overwriting a concurrent
1328                // retirement or publication transaction.
1329                self.state.resource_block = Some(crate::run::ResourceBlock {
1330                    cause: crate::run::ResourceBlockCause::Other,
1331                    node: "start".into(),
1332                    kind: "worktree maintenance".into(),
1333                    detail: detail.clone(),
1334                    at: Timestamp::now(),
1335                });
1336                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1337            }
1338        };
1339        let disk = RunState::load(&self.state.id)?;
1340        crate::pr_head::ensure_owner(&disk)?;
1341        let claimed =
1342            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1343        // Only proven-live ownership refuses here. Unknown records (legacy runs
1344        // with no claim marker or process identity) fall through to the
1345        // exclusive execution lease taken just below, which is what actually
1346        // keeps a second driver out.
1347        anyhow::ensure!(
1348            disk.liveness(claimed) != crate::run::Liveness::Live,
1349            "run already has a live driver; wait for its execution to end"
1350        );
1351        if let Some(to) = &disk.released_to {
1352            bail!(
1353                "run {} cannot continue: its worktree was released to run {}",
1354                self.state.short(),
1355                crate::run::short_of(to)
1356            );
1357        }
1358        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1359        let mut execution = DriverExecution {
1360            runner: self,
1361            driver,
1362        };
1363        // Declare after the execution guard: on any claim error this short
1364        // lock must drop before that guard reloads and releases ownership.
1365        let _owner = owner;
1366        let state = &mut execution.runner.state;
1367        // Moving again, so it is no longer parked. Set before the walk rather
1368        // than in `resume`, so every way of re-entering the graph clears it
1369        // and a card cannot claim a run is waiting to be resumed while the
1370        // agents are already working.
1371        state.parked = false;
1372        state.resource_wait = None;
1373        state.resource_block = None;
1374        // Any seat this state still lists as answering belongs to whatever
1375        // process last drove this run — this one included, if it crashed
1376        // mid-wave. Cleared and flushed immediately, before anything else
1377        // runs, so a resume can never show a seat as live when nothing is
1378        // asking it anything yet; the node that actually dispatches the next
1379        // wave repopulates it.
1380        state.clear_active();
1381        // Recorded in the same spot, and flushed together with the clear
1382        // above: this is the pid a reader checks (`RunState::liveness`) when
1383        // no daemon claim exists to answer "is a process still driving this
1384        // run" — a plain `magi run` / `magi review` typed into a terminal
1385        // claims nothing there. Always overwritten, never only-if-absent, so
1386        // a resumed run's stale pid from a previous, possibly-dead process
1387        // can never survive into this one's own report. Unlike
1388        // `clear_active`, this changes on every single `execute()` call, so
1389        // the save below is now unconditional rather than only-if-cleared.
1390        //
1391        // `driver_started_at` is recorded in the same breath, from this same
1392        // pid, so `liveness` can tell a live pid that is genuinely still us
1393        // apart from one the OS has since handed to an unrelated process —
1394        // see that field's own doc for why the pid alone is not enough.
1395        let pid = std::process::id();
1396        state.driver_pid = Some(pid);
1397        state.driver_started_at = crate::proc::process_started_at(pid);
1398        state.driver_exited = false;
1399        state.driver_claimed = Some(true);
1400        state.begin_pass();
1401        execution.driver.bind(state);
1402        state.save()?;
1403        execution.driver.record(state)?;
1404        Ok(execution)
1405    }
1406
1407    async fn execute_graph(&mut self) -> Result<()> {
1408        self.state.configure_recovery_resume()?;
1409        // Preservation may invalidate an old empty classification. Resolve it
1410        // before any resume shortcut can reuse a verdict or publish a branch.
1411        self.persist_candidates().await?;
1412        if self
1413            .state
1414            .merge_decision
1415            .as_ref()
1416            .is_some_and(|decision| !decision.closed)
1417            && self.state.status != RunStatus::Landing
1418        {
1419            if crate::merge_decision::resumable_hold(&self.state) {
1420                self.state.merge = Some(MergeOutcome {
1421                    mode: MergeMode::Pr,
1422                    ok: false,
1423                    detail: self
1424                        .state
1425                        .merge_decision
1426                        .as_ref()
1427                        .unwrap()
1428                        .target
1429                        .pr
1430                        .clone(),
1431                    empty: false,
1432                });
1433                self.run_land().await?;
1434                self.settle_questions();
1435                return Ok(());
1436            }
1437            bail!(
1438                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1439            );
1440        }
1441        if self.state.config.graph.selected_implementer.is_some()
1442            && self.state.status == RunStatus::Blocked
1443            && self.state.tally.is_none()
1444            && self.state.candidate_preservation.is_empty()
1445            && self.state.interrupted_implementations.is_empty()
1446        {
1447            bail!(
1448                "selected implementation is held; explicitly release the owning task to retry it"
1449            );
1450        }
1451
1452        if self.state.merge_repairs.last().is_some_and(|r| {
1453            matches!(
1454                r.phase,
1455                crate::run::MergeRepairPhase::Requested
1456                    | crate::run::MergeRepairPhase::Fixing
1457                    | crate::run::MergeRepairPhase::Reviewing
1458            )
1459        }) {
1460            if self.repair_merge_review().await? {
1461                self.run_land().await?;
1462            }
1463            self.settle_questions();
1464            return Ok(());
1465        }
1466        // Publication already passed review and gate. Resume only that node:
1467        // syncing to a newer base here could change the verified commit.
1468        if self.state.pr_publication_pending {
1469            self.merge().await?;
1470            if self.resource_halt()? {
1471                return Ok(());
1472            }
1473            self.state.save()?;
1474            self.settle_questions();
1475            return Ok(());
1476        }
1477        // A run that already lost its quorum never resumes into the verdict
1478        // machinery: `deliberate` and `vote` would otherwise clobber the
1479        // stalled marker back to Voting and the run would keep going past a
1480        // verdict that is no longer trustworthy. Everything already recorded is
1481        // kept, so the run stays resumable (or foldable) for a human to pick up.
1482        //
1483        // On --resume the run gets one chance to repair itself: the seats a
1484        // rate limit took out are re-asked. If their quota has since reset and
1485        // the quorum is restored, the run picks up and finishes; otherwise it
1486        // stays stale and still-resumable for a later retry. If it does not
1487        // recover, the returned status stays `Stalled` and nothing was
1488        // clobbered (the recovery only mutates entries for the lost seats).
1489        if self.state.status == RunStatus::Stalled {
1490            if self.recover_stall().await? {
1491                self.finish_after_tally().await?;
1492            } else {
1493                // Still below quorum: persist the marker and stay resumable.
1494                self.state.save()?;
1495            }
1496            return Ok(());
1497        }
1498        // A run parked inside `land` - watching CI, mid fix-round, or
1499        // waiting on the owner's merge approval - resumes directly into it,
1500        // never back through `prep`. Everything before `merge` already
1501        // concluded; that is the only way `status` reaches `Landing` in the
1502        // first place. Re-walking `review_loop` first would also be actively
1503        // wrong: its own status recomputation (see its doc) treats any
1504        // clean round as reason to set `status` to `Gating`, which would
1505        // clobber this marker before `merge` ever ran, and this run would
1506        // never find its way back into `land` at all.
1507        if self.state.status == RunStatus::Landing
1508            || crate::handover::landing_checkpoint(&self.state)
1509        {
1510            self.state.land_sync_pending = true;
1511            self.state.save()?;
1512            self.run_land().await?;
1513            // `run_land` may have settled the run right here - CI came back
1514            // green and the PR merged, say - without ever passing back
1515            // through `merge`'s own trailing call. Whatever it left `status`
1516            // as is what this has to read.
1517            self.settle_questions();
1518            return Ok(());
1519        }
1520        // A run parked at `merge` on the owner's word about a withheld PR text
1521        // resumes straight into `merge`. Re-walking the tail would sync to the
1522        // base again (possibly rebasing, possibly spending the fixer) after the
1523        // gate had passed, and then push a tree nobody reviewed or gated.
1524        if self.state.github_text.is_some()
1525            && self.state.merge.is_none()
1526            && self.state.status == RunStatus::Gating
1527        {
1528            self.merge().await?;
1529            if self.resource_halt()? {
1530                return Ok(());
1531            }
1532            self.state.save()?;
1533            return Ok(());
1534        }
1535        if let Err(e) = self.prep().await {
1536            if e.is::<crate::repo_lock::ResourceBlocked>() {
1537                return Err(e);
1538            }
1539            self.state.status = RunStatus::Blocked;
1540            self.state
1541                .prep_failure
1542                .get_or_insert_with(|| format!("{e:#}"));
1543            self.state
1544                .event("prep", format!("preparation stopped: {e:#}"));
1545            self.state.save()?;
1546            return Err(e);
1547        }
1548        if self.park_here()? {
1549            return Ok(());
1550        }
1551        self.advise().await?;
1552        if self.park_here()? {
1553            return Ok(());
1554        }
1555        self.implement().await?;
1556        if self.park_here()? {
1557            return Ok(());
1558        }
1559        // `after_implement` already saved the state and settled any open
1560        // questions when it set this; nothing later in the graph has
1561        // anything to judge.
1562        if self.state.status == RunStatus::VerifiedNoop {
1563            return Ok(());
1564        }
1565        self.judge().await?;
1566        if self.park_here()? {
1567            return Ok(());
1568        }
1569        self.deliberate().await?;
1570        if self.park_here()? {
1571            return Ok(());
1572        }
1573        self.vote().await?;
1574        if self.park_here()? {
1575            return Ok(());
1576        }
1577        self.tally()?;
1578        // A verdict that lost its quorum is not trustworthy: do not review,
1579        // gate, or merge on it. Everything already done is kept, so the run
1580        // stays resumable (or foldable); the human can replace the agent that
1581        // ran out of quota and pick it up.
1582        if self.state.status == RunStatus::Stalled {
1583            // Persist the stalled marker now — the normal end-of-execute save
1584            // below is below this early return, and without it a resumed run
1585            // would reload a pre-tally status and keep going.
1586            self.state.save()?;
1587            return Ok(());
1588        }
1589        self.finish_after_tally().await?;
1590        Ok(())
1591    }
1592
1593    /// A wave could not start its agents because a shared resource stayed busy
1594    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1595    /// resumable. Unstarted seats have no result; completed peers retain
1596    /// their saved candidate state.
1597    fn stop_for_resource(&mut self) -> Result<()> {
1598        // Resource acquisition observes cancellation inside a node. Complete
1599        // the same durable park as every other node boundary before returning.
1600        if self.pause.parked() || self.interrupt.parked() {
1601            self.park_here()?;
1602            return Ok(());
1603        }
1604        if self.state.resource_block.is_some() {
1605            self.state.status = RunStatus::Blocked;
1606        }
1607        self.state.save()
1608    }
1609
1610    /// Stop walking the graph if a node ended on a resource block.
1611    fn resource_halt(&mut self) -> Result<bool> {
1612        if self.state.resource_block.is_none() {
1613            return Ok(false);
1614        }
1615        self.stop_for_resource()?;
1616        Ok(true)
1617    }
1618
1619    /// Park here if asked to, recording it in the run's own timeline.
1620    ///
1621    /// Returns whether the caller should stop walking the graph. The state is
1622    /// saved either way by the node that just finished; this adds the event so
1623    /// the operator's card says why a run that is neither finished nor moving
1624    /// is sitting where it is.
1625    fn park_here(&mut self) -> Result<bool> {
1626        let parking = self.pause.parked() || self.interrupt.parked();
1627        if !parking && self.state.resource_block.is_some() {
1628            self.stop_for_resource()?;
1629            return Ok(true);
1630        }
1631        // Either handle asking is enough - see `Pause`'s own doc for why
1632        // they are never the same one. `interrupt` is checked second so a
1633        // reason it carries is preferred in the message below over a plain
1634        // shutdown park racing it at the same boundary.
1635        if !self.pause.parked() && !self.interrupt.parked() {
1636            return Ok(false);
1637        }
1638        if self.state.parked {
1639            return Ok(true);
1640        }
1641        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1642        if let Some(block) = &mut self.state.resource_block
1643            && matches!(
1644                block.cause,
1645                crate::run::ResourceBlockCause::Cancelled { .. }
1646            )
1647        {
1648            block.cause = crate::run::ResourceBlockCause::Cancelled {
1649                park: cause.clone(),
1650            };
1651        }
1652        let why = match cause.as_ref().map(ParkCause::label) {
1653            Some(reason) => format!(
1654                "parked after `{}` ({reason}) — resume to carry on from here",
1655                self.state.status.as_str()
1656            ),
1657            None => format!(
1658                "parked after `{}` — resume to carry on from here",
1659                self.state.status.as_str()
1660            ),
1661        };
1662        self.state.event("park", why);
1663        self.state
1664            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1665        self.state.parked = true;
1666        self.state.save()?;
1667        Ok(true)
1668    }
1669
1670    /// Only a new explicit operator approval may retry failed preparation.
1671    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1672        if self.state.prep_failure.is_none() {
1673            return Ok(());
1674        }
1675        let failed_at = self
1676            .state
1677            .events
1678            .iter()
1679            .rev()
1680            .find(|event| event.node == "prep")
1681            .map_or(self.state.updated_at, |event| event.at);
1682        anyhow::ensure!(
1683            approval_at > failed_at,
1684            "preparation retry requires a new explicit release after the failure"
1685        );
1686        self.state.prep_failure = None;
1687        self.state.prep_complete = Some(false);
1688        self.state
1689            .event("prep-retry", "operator approved retry of preparation");
1690        Ok(())
1691    }
1692
1693    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1694    pub fn on_pause(&mut self, pause: Pause) {
1695        self.pause = pause;
1696    }
1697
1698    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1699    /// scheduler asking this one run - and no other - to park so a task
1700    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1701    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1702    /// handle.
1703    pub fn watch_interrupt(&mut self, pause: Pause) {
1704        self.interrupt = pause;
1705    }
1706
1707    /// Abandon this run's own open questions, once `status` has actually
1708    /// settled rather than merely paused.
1709    ///
1710    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1711    /// either back up with the candidates, the review round and the seat
1712    /// sessions already on disk, so a question an implementer asked mid-round
1713    /// may still get a real answer read by a real resume. Only the statuses
1714    /// `resumable` excludes are actually final: the run merged, it reached
1715    /// `Ready` with nothing left to do, it failed outright with no
1716    /// established point to continue from, or every candidate agreed, with
1717    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1718    /// every one of those the seat that asked is gone for good, exactly like
1719    /// the run being deleted under `magi run rm` - so the same cleanup
1720    /// applies, worded for what actually happened instead of "the run was
1721    /// deleted".
1722    ///
1723    /// Best-effort and silent on success: called from every place `status`
1724    /// can land on one of those three, including ones a resumed run revisits,
1725    /// so it must cost nothing when there was nothing open to begin with.
1726    fn settle_questions(&mut self) {
1727        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1728            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1729        }
1730    }
1731
1732    /// The tail of the graph after a trustworthy tally: fold losers, review,
1733    /// gate, merge, and persist.
1734    async fn finish_after_tally(&mut self) -> Result<()> {
1735        // Before the losers are folded: their branches are what the graft
1736        // reads from.
1737        self.graft().await?;
1738        if self.resource_halt()? {
1739            return Ok(());
1740        }
1741        self.fold_losers().await?;
1742        // Before review starts, and again right before the gate: a run's
1743        // review rounds can themselves take long enough for the base to move
1744        // a second time, and the gate is the one node whose "green" gets
1745        // acted on.
1746        self.sync_to_base().await?;
1747        if self.resource_halt()? {
1748            return Ok(());
1749        }
1750        if self.state.status == RunStatus::AlreadyInBase {
1751            return Ok(());
1752        }
1753        self.review_loop().await?;
1754        if self.resource_halt()? {
1755            return Ok(());
1756        }
1757        self.sync_to_base().await?;
1758        if self.resource_halt()? {
1759            return Ok(());
1760        }
1761        if self.state.status == RunStatus::AlreadyInBase {
1762            return Ok(());
1763        }
1764        self.gate().await?;
1765        self.merge().await?;
1766        if self.resource_halt()? {
1767            return Ok(());
1768        }
1769        self.state.save()?;
1770        Ok(())
1771    }
1772
1773    // ---------------------------------------------------------------- prep
1774
1775    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1776    /// run with the step and its output in the event; it is never swallowed.
1777    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1778    /// resume starts over instead of finding candidates and missing seats.
1779    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1780        let result = match ensure_setup_config(&mut self.state, repo).await {
1781            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1782            Err(e) => Err(e),
1783        };
1784        if let Err(e) = result {
1785            git::worktree_remove_clean(repo, wt).await.ok();
1786            if in_prep {
1787                self.rollback_prep(repo).await;
1788            }
1789            self.state.status = RunStatus::Blocked;
1790            self.state.event(
1791                "setup",
1792                format!("worktree setup failed in {}: {e:#}", wt.display()),
1793            );
1794            self.state.save()?;
1795            return Err(e);
1796        }
1797        Ok(())
1798    }
1799
1800    /// Undo everything `prep` created, so a resume re-runs it whole.
1801    async fn rollback_prep(&mut self, repo: &Path) {
1802        let root = self.state.worktree_root();
1803        for c in std::mem::take(&mut self.state.candidates) {
1804            if matches!(
1805                git::worktree_remove_clean(repo, &c.worktree).await,
1806                Ok(true)
1807            ) {
1808                git::branch_delete(repo, &c.branch).await.ok();
1809            } else {
1810                self.state.candidates.push(c);
1811            }
1812        }
1813        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1814        for j in 1..=seats {
1815            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1816                .await
1817                .ok();
1818        }
1819        for k in 1..=self.state.config.graph.advisors {
1820            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1821                .await
1822                .ok();
1823        }
1824    }
1825
1826    async fn prep(&mut self) -> Result<()> {
1827        if let Some(reason) = &self.state.prep_failure {
1828            bail!(
1829                "candidate preparation previously failed: {reason}; correct the source before retrying"
1830            );
1831        }
1832        if self
1833            .state
1834            .prep_complete
1835            .unwrap_or(!self.state.candidates.is_empty())
1836        {
1837            return Ok(());
1838        }
1839        let repo = self.state.repo.clone();
1840        // Includes rollback, config, refs, checkout creation and setup. No
1841        // repository mutation may precede the real shared lock acquisition.
1842        let _coordination = self.coordinate(&repo).await?;
1843        let base = self.state.base_commit.clone();
1844        let plan = self.state.sources.plan(&repo, &base).await?;
1845        self.rollback_prep(&repo).await;
1846        if !self.state.candidates.is_empty() {
1847            bail!(
1848                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1849            );
1850        }
1851        self.state.prep_complete = Some(false);
1852        self.state.status = RunStatus::Prep;
1853        self.state.save()?;
1854        let repo = self.state.repo.clone();
1855        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1856        let root = self.state.worktree_root();
1857        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1858
1859        // The hook is the write-time half of the blindness contract; the
1860        // presentation filter in `blind` is the half that cannot be bypassed.
1861        let hooks_dir = self.state.dir().join("hooks");
1862        if self.state.config.blind.commit_msg_hook {
1863            std::fs::create_dir_all(&hooks_dir)
1864                .with_context(|| format!("create {}", hooks_dir.display()))?;
1865            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1866            let path = hooks_dir.join("commit-msg");
1867            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1868            make_executable(&path)?;
1869            // Ref-counted rather than a plain idempotent set: with more than
1870            // one run able to be in flight in the same repository at once
1871            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1872            // true?" check cannot tell "another run of mine still needs
1873            // this" from "nobody does", and the run that happens to finish
1874            // first would disable the hook out from under a sibling still
1875            // relying on it.
1876            git::acquire_worktree_config(&repo).await?;
1877            self.state.enabled_worktree_config = true;
1878        }
1879
1880        // `[worktree] setup` hides its products per worktree, which needs the
1881        // same per-worktree config the hook does. Held until fold, like it.
1882        ensure_setup_config(&mut self.state, &repo).await?;
1883
1884        for (index, (spec, label)) in self
1885            .roles
1886            .implementers
1887            .clone()
1888            .into_iter()
1889            .zip(labels)
1890            .enumerate()
1891        {
1892            let branch = self.state.branch_for(label);
1893            let worktree = root.join(format!("cand-{label}"));
1894            // Persist intent before Git can leave a partially created tree.
1895            // Candidate registration follows setup, so it cannot inventory
1896            // crashes or refusals during creation itself.
1897            self.state.event(
1898                "prep-intent",
1899                serde_json::json!({
1900                    "branch": branch,
1901                    "worktree": worktree,
1902                    "start": start,
1903                    "picks": plan.picks,
1904                })
1905                .to_string(),
1906            );
1907            self.state.save()?;
1908            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1909            if self.state.config.blind.commit_msg_hook {
1910                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1911            }
1912            git::local_exclude(&worktree, "/.magi/").await?;
1913            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1914                // Nothing of this prep may survive: a resume must start over,
1915                // not find half the candidates and skip the rest.
1916                git::worktree_remove_clean(&repo, &worktree).await.ok();
1917                git::branch_delete(&repo, &branch).await.ok();
1918                self.rollback_prep(&repo).await;
1919                self.state.status = RunStatus::Blocked;
1920                self.state
1921                    .event("prep", format!("worktree setup failed: {e:#}"));
1922                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1923                self.state.save()?;
1924                return Err(e);
1925            }
1926            self.state.candidates.push(Candidate {
1927                index,
1928                label,
1929                agent: spec.id.clone(),
1930                branch,
1931                worktree,
1932                summary: String::new(),
1933                stat: String::new(),
1934                files: 0,
1935                commits: 0,
1936                empty: false,
1937                failed: None,
1938                verified_noop: None,
1939                duration_ms: 0,
1940                folded: false,
1941            });
1942            self.state.save()?;
1943            for pick in &plan.picks {
1944                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1945                {
1946                    self.state.status = RunStatus::Blocked;
1947                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1948                    self.state
1949                        .event("prep", format!("cannot apply referenced commit: {e}"));
1950                    self.state.save()?;
1951                    return Err(e);
1952                }
1953            }
1954        }
1955
1956        for j in 1..=self.roles.judges.len() {
1957            let wt = root.join(format!("judge-{j}"));
1958            if !wt.exists() {
1959                git::worktree_add_detached(&repo, &wt, &base).await?;
1960                self.setup_seat_worktree(&repo, &wt, true).await?;
1961            }
1962        }
1963
1964        // Disposable, detached checkouts for the design-deliberation stage's
1965        // advisor seats — the same shape as the judges' above, at the same
1966        // base commit, since advisors also only ever read. Sized off the
1967        // configured count directly rather than a resolved roster: unlike
1968        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1969        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1970        // `prep` has no `ResolvedRoles` field to read a count from here.
1971        if self.state.config.graph.advise {
1972            for k in 1..=self.state.config.graph.advisors {
1973                let wt = root.join(format!("advisor-{k}"));
1974                if !wt.exists() {
1975                    git::worktree_add_detached(&repo, &wt, &base).await?;
1976                    self.setup_seat_worktree(&repo, &wt, true).await?;
1977                }
1978            }
1979        }
1980
1981        // A judge cannot tell it is looking at its own patch — the seats keep
1982        // separate conversations — but a panel that shares agents with the
1983        // field is less independent than it looks, and that is worth saying out
1984        // loud once per run rather than leaving it in the config.
1985        let authors: Vec<&str> = self
1986            .roles
1987            .implementers
1988            .iter()
1989            .map(|a| a.id.as_str())
1990            .collect();
1991        let overlap: Vec<String> = self
1992            .roles
1993            .judges
1994            .iter()
1995            .enumerate()
1996            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1997            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1998            .collect();
1999        if !overlap.is_empty() {
2000            let note = format!(
2001                "{} also authored a candidate; blind, but the panel is less \
2002                 independent than {} distinct agents would be",
2003                overlap.join(", "),
2004                self.roles.judges.len()
2005            );
2006            self.state.event("prep", note);
2007        }
2008
2009        self.state.event(
2010            "prep",
2011            format!(
2012                "{} candidates, {} judges, base {} ({})",
2013                self.state.candidates.len(),
2014                self.roles.judges.len(),
2015                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2016                self.state.base_branch
2017            ),
2018        );
2019        self.state.prep_complete = Some(true);
2020        self.state.status = RunStatus::Implementing;
2021        self.state.save()?;
2022        Ok(())
2023    }
2024
2025    // -------------------------------------------------------------- advise
2026
2027    /// The design-deliberation stage: independent, read-only advisor seats
2028    /// each sketch a design before any implementer touches the repository,
2029    /// and (when at least one produced a usable proposal) a synthesis seat
2030    /// blends them into a brief `implement` carries in every candidate's
2031    /// prompt.
2032    ///
2033    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2034    /// advisors` is the proposal count. Everything here is best-effort and
2035    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2036    /// that cannot reach quota, or a synthesis seat that produced nothing
2037    /// usable all leave `implement` exactly as it was before this stage
2038    /// existed — the task instruction alone — rather than failing the whole
2039    /// competition over an enrichment stage. Every outcome is still recorded
2040    /// as an event, so a run that got nothing from this stage says why.
2041    ///
2042    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2043    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2044    /// resumed run whose stage failed would re-run it, and re-spend the
2045    /// agent calls, on every reentry before `implement`.
2046    ///
2047    /// Also skipped once any candidate shows implementation progress — the
2048    /// exact predicate `implement` itself uses to decide a candidate is no
2049    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2050    /// is not enough: a run created by an older binary that predates this
2051    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2052    /// an already-`Implementing`-or-later run under this build would
2053    /// otherwise walk straight back through `prep` (a no-op once candidates
2054    /// exist) into this node and spawn every advisor seat against worktrees
2055    /// `prep` never recreated — after implementation has already started,
2056    /// which is exactly the invariant this stage exists to guarantee.
2057    async fn advise(&mut self) -> Result<()> {
2058        let implement_untouched = self
2059            .state
2060            .candidates
2061            .iter()
2062            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2063        if !self.state.config.graph.advise || self.state.advise_attempted {
2064            return Ok(());
2065        }
2066        if !implement_untouched {
2067            self.state.event(
2068                "advise",
2069                "skipping the design-deliberation stage: at least one \
2070                 candidate already shows implementation progress, so this \
2071                 run is past the point the stage exists to run before"
2072                    .to_owned(),
2073            );
2074            self.state.advise_attempted = true;
2075            self.state.save()?;
2076            return Ok(());
2077        }
2078        let run_id = self.state.id.clone();
2079        let prompts = self.state.config.prompts.clone();
2080        let instruction = self.state.instruction.clone();
2081        let language = self.state.config.graph.language.clone();
2082        let root = self.state.worktree_root();
2083        let n = self.state.config.graph.advisors;
2084        let where_recorded = self.state.dir().join("run.json");
2085
2086        let seats = match self.state.config.advisors() {
2087            Ok(seats) if !seats.is_empty() => seats,
2088            Ok(_) => {
2089                self.state.event(
2090                    "advise",
2091                    format!(
2092                        "[graph] advisors is 0; skipping the design-deliberation \
2093                         stage and continuing without a synthesis brief (see {})",
2094                        where_recorded.display()
2095                    ),
2096                );
2097                self.state.advise_attempted = true;
2098                self.state.save()?;
2099                return Ok(());
2100            }
2101            Err(e) => {
2102                self.state.event(
2103                    "advise",
2104                    format!(
2105                        "could not resolve advisor seats ({e:#}); continuing \
2106                         without a design-deliberation brief (see {})",
2107                        where_recorded.display()
2108                    ),
2109                );
2110                self.state.advise_attempted = true;
2111                self.state.save()?;
2112                return Ok(());
2113            }
2114        };
2115
2116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2117        let artifacts = agent::artifacts_dir(&self.state.dir());
2118        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2119
2120        let mut jobs = Vec::new();
2121        for (i, spec) in seats.iter().cloned().enumerate() {
2122            let seat_key = format!("advisor-{}", i + 1);
2123            let seat = self.seat(&seat_key, &spec.id);
2124            jobs.push(SeatJob {
2125                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2126                spec,
2127                seat,
2128                cwd: worktrees[i % worktrees.len()].clone(),
2129                timeout,
2130                allow_write: false,
2131                sessions: false,
2132                artifacts: artifacts.clone(),
2133                stem: seat_key,
2134                handover: None,
2135            });
2136        }
2137
2138        self.state.event(
2139            "advise",
2140            format!(
2141                "{} advisor seat(s) sketching a design in parallel",
2142                jobs.len()
2143            ),
2144        );
2145        let mut quota_losses = Vec::new();
2146        let cache = self.state.config.cache_dir();
2147        let ctx = WaveCtx {
2148            carry_seats: false,
2149            pauses: Some((&self.pause, &self.interrupt)),
2150            run: &run_id,
2151            node: "advise",
2152            prompts: &prompts,
2153            cache: cache.as_deref(),
2154            round: None,
2155        };
2156        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2157        let results = ask_json_wave::<Proposal>(
2158            jobs,
2159            Arc::clone(&self.sem),
2160            self.state.config.graph.retries,
2161            &advisor_roster,
2162            &ctx,
2163            &mut quota_losses,
2164            &mut self.state,
2165            &|p: &Proposal| p.validate(),
2166        )
2167        .await;
2168        if self.state.resource_block.is_some() {
2169            return self.stop_for_resource();
2170        }
2171        self.state.quota.extend(quota_losses);
2172
2173        let mut records = Vec::with_capacity(results.len());
2174        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2175            let agent_id = seat.agent.clone();
2176            self.state.seats.insert(seat.key.clone(), seat);
2177            match res {
2178                Ok((proposal, out)) => {
2179                    self.state
2180                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2181                    records.push(advise::AdvisorRecord::proposed(
2182                        i + 1,
2183                        agent_id,
2184                        proposal,
2185                        out.duration_ms,
2186                    ));
2187                }
2188                Err(e) => {
2189                    self.state.event(
2190                        "advise",
2191                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2192                    );
2193                    records.push(advise::AdvisorRecord::failed(
2194                        i + 1,
2195                        agent_id,
2196                        e.to_string(),
2197                    ));
2198                }
2199            }
2200        }
2201
2202        let mut advice = advise::Advice {
2203            records,
2204            synthesis: None,
2205        };
2206        if advice.proposals().is_empty() {
2207            self.state.event(
2208                "advise",
2209                "no advisor produced a usable proposal; continuing without a \
2210                 synthesis brief"
2211                    .to_owned(),
2212            );
2213        } else {
2214            match self
2215                .synthesize_brief(
2216                    &advice,
2217                    &instruction,
2218                    &language,
2219                    &worktrees[0],
2220                    &artifacts,
2221                    &run_id,
2222                    &prompts,
2223                    cache.as_deref(),
2224                )
2225                .await
2226            {
2227                Ok(Some(text)) => {
2228                    self.state.event(
2229                        "advise",
2230                        "synthesized a design brief for the implementer".to_owned(),
2231                    );
2232                    advice.synthesis = Some(text);
2233                }
2234                Ok(None) => {
2235                    self.state.event(
2236                        "advise",
2237                        "the synthesis seat produced nothing usable; continuing \
2238                         without a design brief"
2239                            .to_owned(),
2240                    );
2241                }
2242                Err(e) => {
2243                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2244                        return Err(e);
2245                    }
2246                    self.state.event(
2247                        "advise",
2248                        format!("could not synthesize a design brief: {e:#}"),
2249                    );
2250                }
2251            }
2252        }
2253        advise::apply_reflection(&mut advice);
2254
2255        self.state.advice = Some(advice);
2256        self.state.advise_attempted = true;
2257        self.state.save()?;
2258        Ok(())
2259    }
2260
2261    /// The synthesis seat: reads every advisor's proposal and blends them
2262    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2263    /// out of [`Runner::advise`] only for readability — it is not called
2264    /// anywhere else.
2265    ///
2266    /// Picked the same way [`crate::talk`]'s standing conversation and
2267    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2268    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2269    /// order (a claude seat, else the first runnable agent in roster order)
2270    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2271    /// in [`crate::config`] for why a dedicated field exists here at all.
2272    #[allow(clippy::too_many_arguments)]
2273    async fn synthesize_brief(
2274        &mut self,
2275        advice: &advise::Advice,
2276        instruction: &str,
2277        language: &str,
2278        cwd: &Path,
2279        artifacts: &Path,
2280        run_id: &str,
2281        prompts: &Prompts,
2282        cache: Option<&Path>,
2283    ) -> Result<Option<String>> {
2284        let chain = agent::pick_chain(
2285            &self.state.config.agents,
2286            self.state.config.roles.synthesizer.as_ref(),
2287            &agent::installed,
2288            "synthesizer",
2289        )?;
2290        let proposals = advice.proposals();
2291        let mut prompt = prompt::with_overlay(
2292            prompt::synthesize_brief(instruction, &proposals, language),
2293            prompts.overlay("advise"),
2294        );
2295        if cache.is_some() {
2296            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2297            // below — see `prompt::build_cache_note`'s doc for why telling a
2298            // read-only seat to build through the shared cache is exactly how
2299            // a sandbox's write refusal gets misread as a defect.
2300            prompt.push('\n');
2301            prompt.push_str(&prompt::build_cache_note("advise", false));
2302        }
2303        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2304        // Each id is tried once, in order; a quota hit, error or unusable
2305        // answer moves to the next. The seat is single-turn (`sessions:
2306        // false`) and the prompt is the whole context, so a fallback agent
2307        // needs nothing carried over.
2308        let mut last = None;
2309        for (n, spec) in chain.iter().enumerate() {
2310            if n > 0 {
2311                self.state
2312                    .event("advise", format!("synthesis falling back to {}", spec.id));
2313            }
2314            let mut seat = self.seat("advise-synthesis", &spec.id);
2315            let wait = self.repository_wait();
2316            let outcome = wait
2317                .scope(agent::invoke(
2318                    spec,
2319                    &mut seat,
2320                    &Invocation {
2321                        scope: crate::agent::InvocationScope::Repository,
2322                        coordination: self.state.config.coordination.as_ref(),
2323                        fetched_base: None,
2324                        cwd,
2325                        prompt: &prompt,
2326                        timeout,
2327                        allow_write: false,
2328                        unsandboxed: false,
2329                        sessions: false,
2330                        artifacts,
2331                        stem: &if n == 0 {
2332                            "advise-synthesis".to_owned()
2333                        } else {
2334                            format!("advise-synthesis-{}", spec.id)
2335                        },
2336                        run: run_id,
2337                        node: "advise",
2338                        cache_dir: None,
2339                        attachments: &[],
2340                        writable: &[],
2341                    },
2342                ))
2343                .await;
2344            if outcome.is_ok() {
2345                self.state.seats.insert(seat.key.clone(), seat);
2346            }
2347            let advance = agent::chain_advances(&outcome);
2348            last = Some(outcome);
2349            if !advance {
2350                break;
2351            }
2352        }
2353        // Exhausted: the last attempt's result is what a single failed seat
2354        // would have produced.
2355        let out = last.expect("a chain holds at least one agent")?;
2356        if !out.usable() {
2357            return Ok(None);
2358        }
2359        let text =
2360            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2361        Ok((!text.trim().is_empty()).then_some(text))
2362    }
2363
2364    // ----------------------------------------------------------- implement
2365
2366    async fn implement(&mut self) -> Result<()> {
2367        self.persist_candidates().await?;
2368        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2369        // agent files with `magi task add` name the run that paid for it. The
2370        // prompt overlay is cloned alongside it because the waves borrow it
2371        // while `self` is mutably borrowed by the node's own bookkeeping.
2372        let run_id = self.state.id.clone();
2373        let prompts = self.state.config.prompts.clone();
2374        let todo: Vec<usize> = self
2375            .state
2376            .candidates
2377            .iter()
2378            .enumerate()
2379            .filter(|(_, c)| {
2380                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2381                    && c.failed.is_none()
2382                    && !c.empty
2383            })
2384            .map(|(i, _)| i)
2385            .collect();
2386        if todo.is_empty() {
2387            return self.after_implement();
2388        }
2389        self.state.status = RunStatus::Implementing;
2390
2391        let language = self.state.config.graph.language.clone();
2392        let github_language = self.state.config.graph.github_language.clone();
2393        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2394        let sessions = self.state.config.graph.sessions;
2395        let artifacts = agent::artifacts_dir(&self.state.dir());
2396        // The design-deliberation stage's blended brief, when `advise` found
2397        // one — carried into every implementer's prompt the same way
2398        // regardless of which candidate it is.
2399        let brief = self
2400            .state
2401            .advice
2402            .as_ref()
2403            .and_then(|a| a.synthesis.as_deref())
2404            .map(str::to_owned);
2405        let attachments = self.state.attachments.clone();
2406
2407        let mut jobs = Vec::new();
2408        for &i in &todo {
2409            let (index, label, worktree) = {
2410                let c = &self.state.candidates[i];
2411                (c.index, c.label, c.worktree.clone())
2412            };
2413            let spec = self.roles.implementers[index].clone();
2414            let seat_key = format!("impl-{label}");
2415            let seat = self.seat(&seat_key, &spec.id);
2416            let instruction = seeded_instruction(&self.state);
2417            jobs.push(SeatJob {
2418                spec,
2419                seat,
2420                prompt: prompt::implement_in(
2421                    &instruction,
2422                    &worktree.to_string_lossy(),
2423                    &language,
2424                    &github_language,
2425                    brief.as_deref(),
2426                    &attachments,
2427                ),
2428                cwd: worktree,
2429                timeout,
2430                allow_write: true,
2431                sessions,
2432                artifacts: artifacts.clone(),
2433                stem: format!("impl-{label}"),
2434                handover: None,
2435            });
2436        }
2437
2438        self.state.event(
2439            "implement",
2440            format!("{} candidates in parallel", jobs.len()),
2441        );
2442        // Kept so a seat whose CLI hung up can be asked again from the same
2443        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2444        // can update a seat's own entry once a fallback agent takes it over —
2445        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2446        // whichever agent actually answered, not the one that quota'd out.
2447        let mut sent = jobs.clone();
2448        let cache = self.state.config.cache_dir();
2449        let ctx = WaveCtx {
2450            carry_seats: false,
2451            pauses: Some((&self.pause, &self.interrupt)),
2452            run: &run_id,
2453            node: "implement",
2454            prompts: &prompts,
2455            cache: cache.as_deref(),
2456            round: None,
2457        };
2458        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2459        if !wave_blocked(&results) {
2460            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2461                .await;
2462            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2463                .await?;
2464            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2465                .await;
2466        }
2467        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2468
2469        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2470            && results
2471                .iter()
2472                .any(|(_, _, out)| FailClass::of(out).is_some());
2473        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2474            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2475                // Executed turns interrupted by disk pressure already carry
2476                // a preservation barrier from wave. Admission-only waits do
2477                // not. Keep the interrupted seat's session for recovery too.
2478                if self
2479                    .state
2480                    .candidate_preservation
2481                    .contains_key(&self.state.candidates[i].label)
2482                {
2483                    self.state.seats.insert(seat.key.clone(), seat);
2484                }
2485                continue;
2486            }
2487            let seat_key = seat.key.clone();
2488            // A quota fallback (`resume_seat_handovers`) may have handed this
2489            // seat to a different agent than the one `prep` recorded on the
2490            // candidate; the stats tables and any later fixer-defaults-to-
2491            // winner's-author lookup must credit whoever actually answered —
2492            // unless every fallback also quota'd out, in which case nobody
2493            // actually answered and crediting the last agent tried would
2494            // erase every earlier agent's own quota loss from the stats
2495            // tables instead of just this one seat's.
2496            let agent = seat.agent.clone();
2497            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2498            self.state.seats.insert(seat.key.clone(), seat);
2499            let label = self.state.candidates[i].label;
2500            self.state.interrupted_implementations.remove(&label);
2501
2502            let (summary, duration, failed, verified_claim) = match out {
2503                AgentOutcome::Ok(o) => {
2504                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2505                    let failed = (!o.usable()).then(|| {
2506                        if o.timed_out {
2507                            "agent timed out".to_owned()
2508                        } else {
2509                            format!(
2510                                "agent exited with {}",
2511                                crate::notices::exit_status(o.exit_code)
2512                            )
2513                        }
2514                    });
2515                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2516                    (text, o.duration_ms, failed, verified_claim)
2517                }
2518                // Left un-resumed by `resume_undelivered` (a dirty tree
2519                // already rescues the work, or there was no session left to
2520                // resume into) — reported like the ordinary failure it is,
2521                // never as if `o.text` (the CLI's raw error JSON) were an
2522                // answer.
2523                AgentOutcome::Dropped(o) => {
2524                    let why = o
2525                        .dropped
2526                        .as_ref()
2527                        .map(|d| d.why.as_str())
2528                        .unwrap_or("the CLI ended the stream without delivering its answer");
2529                    (
2530                        String::new(),
2531                        o.duration_ms,
2532                        Some(format!("the CLI dropped the stream ({why})")),
2533                        None,
2534                    )
2535                }
2536                AgentOutcome::Quota(o) => {
2537                    self.state.quota.push(QuotaLoss {
2538                        seat: seat_key,
2539                        node: "implement".to_owned(),
2540                        at: Timestamp::now(),
2541                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2542                    });
2543                    (
2544                        String::new(),
2545                        o.duration_ms,
2546                        Some("rate limited (quota); produced no change".to_owned()),
2547                        None,
2548                    )
2549                }
2550                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2551                    (String::new(), 0, Some(e), None)
2552                }
2553            };
2554
2555            // Save every completed turn before any fallible Git operation. A failed
2556            // rescue must not erase the reply or prevent collection of other seats.
2557            let c = &mut self.state.candidates[i];
2558            if !exhausted_the_fallback_chain {
2559                c.agent = agent;
2560            }
2561            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2562            c.duration_ms = duration;
2563            c.failed = failed;
2564            c.verified_noop = verified_claim;
2565            self.state
2566                .candidate_preservation
2567                .insert(label, "awaiting persistence".to_owned());
2568        }
2569        self.state.save()?;
2570        self.persist_candidates().await?;
2571
2572        if resource_blocked {
2573            return self.stop_for_resource();
2574        }
2575        if selected_failed {
2576            self.state.status = RunStatus::Blocked;
2577            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2578            self.state.save()?;
2579            bail!(
2580                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2581            );
2582        }
2583        self.after_implement()
2584    }
2585
2586    /// Retry only persistence after a completed implementation turn.
2587    async fn persist_candidates(&mut self) -> Result<()> {
2588        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2589        for label in labels {
2590            let i = self
2591                .state
2592                .candidates
2593                .iter()
2594                .position(|c| c.label == label)
2595                .context("pending candidate is missing")?;
2596            let c = self.state.candidates[i].clone();
2597            let interrupted = self.state.interrupted_implementations.contains(&label);
2598            let result = async {
2599                let wait = crate::repo_lock::WaitPolicy::new(
2600                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2601                    || false,
2602                );
2603                let _coordination =
2604                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2605                // Best effort evidence requires no index writes. A successful snapshot
2606                // never authorizes removal of a tree whose rescue failed.
2607                if let Err(e) = git::snapshot_uncommitted(
2608                    &c.worktree,
2609                    &self
2610                        .state
2611                        .dir()
2612                        .join("artifacts")
2613                        .join(format!("candidate-{label}-recovery")),
2614                )
2615                .await
2616                {
2617                    self.state.event(
2618                        "preserve",
2619                        format!("candidate {label} snapshot incomplete: {e:#}"),
2620                    );
2621                }
2622                let rescued = git::rescue_commit(
2623                    &c.worktree,
2624                    &format!("magi: candidate {label} (uncommitted work)"),
2625                )
2626                .await?;
2627                self.state.note_withheld("implement", &rescued.withheld);
2628                let base = &self.state.base_commit;
2629                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2630                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2631                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2632                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2633                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2634                if c.empty && commits > 0 && !patch.trim().is_empty()
2635                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2636                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2637                        || self.state.tally.is_some())
2638                {
2639                    // Keep the old classification and pending marker so every
2640                    // retry stops too. Retain decisions as recovery evidence;
2641                    // only a supported recovery operation may replace them.
2642                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2643                }
2644                let c = &mut self.state.candidates[i];
2645                c.commits = commits;
2646                c.stat = stat;
2647                c.files = files;
2648                let no_changes = commits == 0 || patch.trim().is_empty();
2649                c.empty = no_changes && !interrupted;
2650                if interrupted && no_changes {
2651                    c.failed = None;
2652                    c.verified_noop = None;
2653                }
2654                if !c.empty {
2655                    c.failed = None;
2656                    c.verified_noop = None;
2657                }
2658                self.state.event(
2659                    "preserve",
2660                    format!("candidate {label}: {files} files, {commits} commits"),
2661                );
2662                Ok::<_, anyhow::Error>(())
2663            }
2664            .await;
2665            match result {
2666                Ok(()) => {
2667                    self.state.candidate_preservation.remove(&label);
2668                    if !interrupted || self.state.candidates[i].files > 0 {
2669                        self.state.interrupted_implementations.remove(&label);
2670                    }
2671                }
2672                Err(e) => {
2673                    let reason = format!("candidate {label} preservation failed: {e:#}");
2674                    self.state.event("preserve", &reason);
2675                    self.state.candidate_preservation.insert(label, reason);
2676                }
2677            }
2678            if self.state.candidate_preservation.is_empty()
2679                && self.state.status == RunStatus::Blocked
2680                && self.state.candidates.iter().all(|c| c.failed.is_none())
2681            {
2682                // Persist the recovered phase with the cleared pending marker;
2683                // a crash between saves must not turn this into a selected-seat
2684                // failure hold on the next resume.
2685                self.state.status = RunStatus::Implementing;
2686                self.state.event(
2687                    "preserve",
2688                    "candidate persistence recovered; implementation can continue",
2689                );
2690            }
2691            self.state.save()?;
2692        }
2693        if !self.state.candidate_preservation.is_empty() {
2694            self.state.status = RunStatus::Blocked;
2695            self.state.save()?;
2696            bail!(
2697                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2698            );
2699        }
2700        Ok(())
2701    }
2702
2703    /// Ask again, once, for work a CLI did and then failed to hand over.
2704    ///
2705    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2706    /// status with an empty response and a usage report showing output tokens,
2707    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2708    /// seven minutes and 14,267 output tokens that arrived as an empty
2709    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2710    ///
2711    /// Two conditions, and both matter:
2712    ///
2713    /// - **Only when the tree is untouched.** Often the agent has already
2714    ///   written its files and only the closing message was lost; the rescue
2715    ///   commit below picks that up and there is nothing to ask for. Re-asking
2716    ///   then would pay for a second implementation of work already on disk.
2717    /// - **Once.** A CLI that drops one stream can drop the next, and this
2718    ///   node is the most expensive in the graph.
2719    ///
2720    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2721    /// dropped reply still carried its `conversation_id`, so the seat is asked
2722    /// to finish what it was doing rather than sent the whole task again. It
2723    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2724    /// node's - for the same reason a re-ranked judge does: restating finished
2725    /// work is not the work.
2726    ///
2727    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2728    /// same way until it resets, while an abandoned conversation is still
2729    /// there to be picked up.
2730    async fn resume_undelivered(
2731        &mut self,
2732        results: &mut [(usize, SeatState, AgentOutcome)],
2733        sent: &[SeatJob],
2734        prompts: &Prompts,
2735        run_id: &str,
2736    ) {
2737        for (wi, seat, out) in results.iter_mut() {
2738            let Some(dropped) = (match &*out {
2739                AgentOutcome::Dropped(o) => o.dropped.clone(),
2740                _ => None,
2741            }) else {
2742                continue;
2743            };
2744            let Some(job) = sent.get(*wi) else { continue };
2745            // Already on disk? Then only the closing message was lost.
2746            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2747                self.state.event(
2748                    "implement",
2749                    format!(
2750                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2751                         work is in the tree",
2752                        seat.key, dropped.output_tokens, dropped.why
2753                    ),
2754                );
2755                continue;
2756            }
2757            // The re-ask only makes sense as a resume: `resume_after_drop`
2758            // says nothing about the task, trusting the seat to still hold it.
2759            // Without a session to resume — sessions disabled, or this CLI's
2760            // drop shape happened not to carry a session id — that prompt
2761            // would open a brand-new conversation with no context at all,
2762            // which is worse than leaving this as the ordinary failure it
2763            // already is.
2764            if !has_context(&job.spec, seat, job.sessions) {
2765                self.state.event(
2766                    "implement",
2767                    format!(
2768                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2769                         is no session left to resume",
2770                        seat.key, dropped.output_tokens, dropped.why
2771                    ),
2772                );
2773                continue;
2774            }
2775            self.state.event(
2776                "implement",
2777                format!(
2778                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2779                     conversation",
2780                    seat.key, dropped.output_tokens, dropped.why
2781                ),
2782            );
2783            let mut retry = job.clone();
2784            retry.seat = seat.clone();
2785            retry.prompt = prompt::resume_after_drop(&dropped.why);
2786            retry.timeout = retry_budget(job.timeout, true);
2787            retry.stem = format!("{}-resume", job.stem);
2788            let cache = self.state.config.cache_dir();
2789            let ctx = WaveCtx {
2790                carry_seats: false,
2791                pauses: Some((&self.pause, &self.interrupt)),
2792                run: run_id,
2793                node: "implement",
2794                prompts,
2795                cache: cache.as_deref(),
2796                round: None,
2797            };
2798            let (resumed_seat, resumed) =
2799                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2800            *seat = resumed_seat;
2801            *out = resumed;
2802        }
2803    }
2804
2805    /// Fall an implement seat through to the next untried agent in the
2806    /// implementer roster when it lost to quota — or, since the handover was
2807    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2808    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2809    /// quota path itself being unchanged — instead of leaving the
2810    /// seat's loss final the moment one agent's account runs dry.
2811    ///
2812    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2813    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2814    /// truncates to the single slot rotation picked, so a solo task whose one
2815    /// implementer hits quota mid-run used to have nothing else to try. This
2816    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2817    /// unrotated roster — which is the only place the *other* candidates in
2818    /// the machine's roster still exist once `implementers` has been cut down
2819    /// to size.
2820    ///
2821    /// Walks forward from just past the seat's own original position in the
2822    /// roster, never wrapping back to the front: a later candidate slot (say
2823    /// `beta`, the roster's second entry) must fall through to the *next*
2824    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2825    /// almost certainly a different candidate's own agent already — and once
2826    /// the roster's tail is exhausted there is nothing left to fall through
2827    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2828    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2829    /// let this retry that id forever. The loop keeps falling through until
2830    /// an attempt lands something other than `Quota` or the roster's tail
2831    /// runs out of untried ids, at which point the seat is left exactly as
2832    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2833    /// `QuotaLoss` recorded, the candidate failed/empty.
2834    ///
2835    /// `sent` is taken mutably and updated with the fallback agent's spec:
2836    /// `resume_unconfirmed_commands`, which runs after this and also reads
2837    /// `sent`, must see whichever agent actually ended up answering the seat
2838    /// — reading the stale, original spec there would check session
2839    /// eligibility against the wrong CLI and could hand a fallback agent's
2840    /// session id to the agent that just lost the seat to quota.
2841    ///
2842    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2843    /// — `self.seat` only reuses state when the agent id is unchanged, so
2844    /// handing it a different id already gets this for free. Reusing the old
2845    /// seat would resume a different CLI's session as if it were a
2846    /// continuation of this one.
2847    ///
2848    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2849    /// a quota loss cuts an agent off mid-turn, so anything already in the
2850    /// tree is unfinished work, not a completed candidate a re-ask would pay
2851    /// for twice. A dirty tree is rescued into a commit first (the same
2852    /// neutral-identity rescue `implement`'s own outcome loop gives every
2853    /// candidate) so the next agent starts clean.
2854    ///
2855    /// The new agent gets the implementer's full prompt and full
2856    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2857    /// it has no session and no context, and is implementing the task from
2858    /// nothing, unlike a resumed drop which is only restating work already
2859    /// done.
2860    ///
2861    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2862    /// `implement` event, never into `self.state.quota` — that is what
2863    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2864    /// go unspent, and a seat that ultimately recovered on its second or
2865    /// third agent is not the stalled panel that check exists to catch. Only
2866    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2867    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2868    /// outcome loop already has — this helper never pushes to it itself.
2869    async fn resume_seat_handovers(
2870        &mut self,
2871        results: &mut [(usize, SeatState, AgentOutcome)],
2872        sent: &mut [SeatJob],
2873        prompts: &Prompts,
2874        run_id: &str,
2875    ) -> Result<()> {
2876        let instruction = seeded_instruction(&self.state);
2877        let language = self.state.config.graph.language.clone();
2878        let github_language = self.state.config.graph.github_language.clone();
2879        let brief = self
2880            .state
2881            .advice
2882            .as_ref()
2883            .and_then(|a| a.synthesis.as_deref())
2884            .map(str::to_owned);
2885        let attachments = self.state.attachments.clone();
2886        for (wi, seat, out) in results.iter_mut() {
2887            // Who holds the other candidate seats of this wave right now
2888            // (earlier handovers already written back to `sent`).
2889            let others: BTreeSet<String> = sent
2890                .iter()
2891                .enumerate()
2892                .filter(|(j, _)| j != wi)
2893                .map(|(_, j)| j.spec.id.clone())
2894                .collect();
2895            let Some(job) = sent.get_mut(*wi) else {
2896                continue;
2897            };
2898            // Where the seat's own original agent sits in the roster — the
2899            // fallback walk starts just past here, never at the front, so a
2900            // later candidate slot's quota loss does not fall back onto an
2901            // earlier slot's own agent.
2902            let start = self
2903                .roles
2904                .implementer_roster
2905                .iter()
2906                .position(|s| s.id == job.spec.id)
2907                .unwrap_or(0);
2908            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2909            let mut fallback_attempt = 0usize;
2910            let mut prev: Option<FailClass> = None;
2911            while let Some(cur) = FailClass::of(&*out) {
2912                if !should_hand_over(prev.as_ref(), &cur) {
2913                    break;
2914                }
2915                let Some(next) =
2916                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2917                        .cloned()
2918                else {
2919                    break;
2920                };
2921                tried.insert(next.id.clone());
2922                fallback_attempt += 1;
2923
2924                let coordination = self.coordinate(&job.cwd).await?;
2925                if let Ok(r) = git::rescue_commit(
2926                    &job.cwd,
2927                    &format!(
2928                        "magi: candidate {} (uncommitted work before {} fallback)",
2929                        seat.key,
2930                        if cur == FailClass::Quota {
2931                            "quota"
2932                        } else {
2933                            "handover"
2934                        }
2935                    ),
2936                )
2937                .await
2938                {
2939                    self.state.note_withheld("implement", &r.withheld);
2940                }
2941                drop(coordination);
2942
2943                record_handover(
2944                    &mut self.state,
2945                    "implement",
2946                    &seat.key,
2947                    &seat.agent,
2948                    &next.id,
2949                    &cur,
2950                    &fail_reason(&*out),
2951                );
2952                prev = Some(cur.clone());
2953
2954                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2955                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2956                // Kept in sync on `sent` itself, not just the local retry: a
2957                // later helper (`resume_unconfirmed_commands`) reads `sent`
2958                // after this one returns and must see whichever agent is now
2959                // occupying the seat, not the one that just quota'd out —
2960                // otherwise it would judge session/continuation eligibility
2961                // by the wrong CLI and could resend a fallback's session id
2962                // to the agent that lost it the seat in the first place.
2963                job.spec = next.clone();
2964                let mut retry = job.clone();
2965                retry.seat = new_seat;
2966                retry.prompt = prompt::implement_in(
2967                    &instruction,
2968                    &job.cwd.to_string_lossy(),
2969                    &language,
2970                    &github_language,
2971                    brief.as_deref(),
2972                    &attachments,
2973                );
2974                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2975                let cache = self.state.config.cache_dir();
2976                let ctx = WaveCtx {
2977                    carry_seats: false,
2978                    pauses: Some((&self.pause, &self.interrupt)),
2979                    run: run_id,
2980                    node: "implement",
2981                    prompts,
2982                    cache: cache.as_deref(),
2983                    round: None,
2984                };
2985                let (fallback_seat, fallback_out) = run_one(
2986                    retry,
2987                    Arc::clone(&self.sem),
2988                    &ctx,
2989                    &mut self.state,
2990                    fallback_attempt,
2991                )
2992                .await;
2993                *seat = fallback_seat;
2994                *out = fallback_out;
2995            }
2996        }
2997        Ok(())
2998    }
2999
3000    /// Ask an implement seat's own CLI to confirm what it started, once, when
3001    /// its reply reported a command whose completion status it never
3002    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3003    /// that does and does not mean.
3004    ///
3005    /// The completion contract this task asks for, extended to `implement`
3006    /// with the same signal `continue_fix_report` reads for the fixer,
3007    /// rather than a keyword search over the reply or a hard requirement on
3008    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3009    /// a candidate's CLI turn ended cleanly while a test run it had started
3010    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3011    /// named in it at all is untouched by this: `commands` is empty, so
3012    /// there is nothing to be unconfirmed.
3013    ///
3014    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3015    /// this is not about recovering edits that might already be on disk, it
3016    /// is about a result the seat itself never vouched for, which resuming
3017    /// asks for regardless of what the tree already holds. Bounded to one
3018    /// attempt for the same reason `resume_undelivered` is — this is the
3019    /// most expensive node in the graph — and a seat that still cannot
3020    /// confirm on that attempt is left as whatever its (possibly still
3021    /// unconfirmed) reply says; this does not invent a new "failed" reason
3022    /// for a candidate that otherwise produced a real, committed change.
3023    async fn resume_unconfirmed_commands(
3024        &mut self,
3025        results: &mut [(usize, SeatState, AgentOutcome)],
3026        sent: &[SeatJob],
3027        prompts: &Prompts,
3028        run_id: &str,
3029    ) {
3030        for (wi, seat, out) in results.iter_mut() {
3031            let AgentOutcome::Ok(o) = &*out else {
3032                continue;
3033            };
3034            if !has_unconfirmed_command(&o.commands) {
3035                continue;
3036            }
3037            let Some(job) = sent.get(*wi) else { continue };
3038            if !has_context(&job.spec, seat, job.sessions) {
3039                self.state.event(
3040                    "implement",
3041                    format!(
3042                        "{}: the reply named a command whose own CLI never confirmed the exit \
3043                         status of, but there is no session left to resume",
3044                        seat.key
3045                    ),
3046                );
3047                continue;
3048            }
3049            self.state.event(
3050                "implement",
3051                format!(
3052                    "{}: the reply named a command whose own CLI never confirmed the exit \
3053                     status of; resuming the conversation",
3054                    seat.key
3055                ),
3056            );
3057            let mut retry = job.clone();
3058            retry.seat = seat.clone();
3059            retry.prompt = prompt::resume_incomplete(
3060                "a command in your last reply had no confirmed exit status",
3061            );
3062            retry.timeout = retry_budget(job.timeout, true);
3063            retry.stem = format!("{}-confirm", job.stem);
3064            let cache = self.state.config.cache_dir();
3065            let ctx = WaveCtx {
3066                carry_seats: false,
3067                pauses: Some((&self.pause, &self.interrupt)),
3068                run: run_id,
3069                node: "implement",
3070                prompts,
3071                cache: cache.as_deref(),
3072                round: None,
3073            };
3074            let (resumed_seat, resumed) =
3075                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3076            *seat = resumed_seat;
3077            *out = resumed;
3078        }
3079    }
3080
3081    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3082    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3083    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3084    /// that motivated this.
3085    ///
3086    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3087    /// own nudge loop already covers for judge/review/vote seats, and not a
3088    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3089    /// implement seats: here the CLI turn genuinely finished while the node's
3090    /// own work — the fixer's account of what it did — had not. Gated purely
3091    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3092    /// reply, never on any wording in it, so a fixer whose valid, first-try
3093    /// `FixReport` happens to mention having waited on a background test is
3094    /// never resumed — the `Ok(report)` branch at the call site returns
3095    /// before this is ever invoked.
3096    ///
3097    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3098    /// attempt ([`retry_budget`]), nothing attempted once the session is
3099    /// gone, and a quota hit ends the loop immediately rather than retrying a
3100    /// rate limit that fails the same way again.
3101    async fn continue_fix_report(
3102        &mut self,
3103        mut seat: SeatState,
3104        parse_err: String,
3105        job: &SeatJob,
3106        prompts: &Prompts,
3107        run_id: &str,
3108        round: usize,
3109    ) -> (
3110        SeatState,
3111        Option<FixReport>,
3112        Option<String>,
3113        ContinuationRecord,
3114    ) {
3115        let mut last_err = parse_err;
3116        let mut cumulative_wait_ms = 0u64;
3117        let mut attempts = 0usize;
3118        loop {
3119            if !has_context(&job.spec, &seat, job.sessions) {
3120                self.state.event(
3121                    "fix",
3122                    format!(
3123                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3124                         session left to resume into"
3125                    ),
3126                );
3127                let outcome = if attempts == 0 {
3128                    ContinuationOutcome::NoSession
3129                } else {
3130                    ContinuationOutcome::Exhausted
3131                };
3132                return (
3133                    seat,
3134                    None,
3135                    Some(format!("unparsable fix report: {last_err}")),
3136                    ContinuationRecord {
3137                        attempts,
3138                        cumulative_wait_ms,
3139                        outcome,
3140                    },
3141                );
3142            }
3143            if attempts >= MAX_FIX_CONTINUATIONS {
3144                self.state.event(
3145                    "fix",
3146                    format!(
3147                        "round {round}: fixer's reply still had no adoption report after \
3148                         {attempts} continuation(s) ({last_err}); giving up"
3149                    ),
3150                );
3151                return (
3152                    seat,
3153                    None,
3154                    Some(format!(
3155                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3156                    )),
3157                    ContinuationRecord {
3158                        attempts,
3159                        cumulative_wait_ms,
3160                        outcome: ContinuationOutcome::Exhausted,
3161                    },
3162                );
3163            }
3164            attempts += 1;
3165            self.state.event(
3166                "fix",
3167                format!(
3168                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3169                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3170                ),
3171            );
3172            let mut retry = job.clone();
3173            retry.seat = seat.clone();
3174            retry.prompt = prompt::resume_incomplete(&last_err);
3175            retry.timeout = retry_budget(job.timeout, true);
3176            retry.stem = format!("{}-continue{attempts}", job.stem);
3177            let cache = self.state.config.cache_dir();
3178            let ctx = WaveCtx {
3179                carry_seats: false,
3180                pauses: Some((&self.pause, &self.interrupt)),
3181                run: run_id,
3182                node: "fix",
3183                prompts,
3184                cache: cache.as_deref(),
3185                round: Some(round),
3186            };
3187            let (resumed_seat, resumed_out) = run_one(
3188                retry,
3189                Arc::clone(&self.sem),
3190                &ctx,
3191                &mut self.state,
3192                attempts,
3193            )
3194            .await;
3195            seat = resumed_seat;
3196            match resumed_out {
3197                AgentOutcome::Ok(o) => {
3198                    cumulative_wait_ms += o.duration_ms;
3199                    match verdict::extract_json::<FixReport>(&o.text) {
3200                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3201                            self.state.event(
3202                                "fix",
3203                                format!(
3204                                    "round {round}: fixer's adoption report recovered after \
3205                                     {attempts} continuation(s)"
3206                                ),
3207                            );
3208                            return (
3209                                seat,
3210                                Some(report),
3211                                None,
3212                                ContinuationRecord {
3213                                    attempts,
3214                                    cumulative_wait_ms,
3215                                    outcome: ContinuationOutcome::Resumed,
3216                                },
3217                            );
3218                        }
3219                        // The report parsed, but this same reply's own
3220                        // CommandEvidence — the identical record `state.jobs`
3221                        // renders — names a command whose CLI never
3222                        // confirmed an exit status. Read together, that is
3223                        // not a resolved answer: keep nudging rather than
3224                        // accept a report standing next to a command the
3225                        // seat's own CLI cannot vouch for.
3226                        Ok(_) => {
3227                            last_err = "the reply parsed, but it reported a command whose own CLI \
3228                                 never confirmed an exit status"
3229                                .to_owned();
3230                        }
3231                        Err(e) => last_err = e.to_string(),
3232                    }
3233                }
3234                AgentOutcome::Quota(o) => {
3235                    cumulative_wait_ms += o.duration_ms;
3236                    self.state.quota.push(QuotaLoss {
3237                        seat: seat.key.clone(),
3238                        node: "fix".to_owned(),
3239                        at: Timestamp::now(),
3240                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3241                    });
3242                    self.state.event(
3243                        "fix",
3244                        format!(
3245                            "round {round}: continuation rate limited (quota); not retrying now"
3246                        ),
3247                    );
3248                    return (
3249                        seat,
3250                        None,
3251                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3252                        ContinuationRecord {
3253                            attempts,
3254                            cumulative_wait_ms,
3255                            outcome: ContinuationOutcome::QuotaLost,
3256                        },
3257                    );
3258                }
3259                AgentOutcome::Dropped(o) => {
3260                    cumulative_wait_ms += o.duration_ms;
3261                    let why = o
3262                        .dropped
3263                        .as_ref()
3264                        .map(|d| d.why.as_str())
3265                        .unwrap_or("the CLI ended the stream without delivering its answer");
3266                    last_err = format!("the CLI dropped the stream ({why})");
3267                }
3268                AgentOutcome::Failed(e) => last_err = e,
3269                // The continuation was never started; the report stays lost
3270                // and the caller records that, without any handover.
3271                AgentOutcome::ResourceBlocked(e) => {
3272                    return (
3273                        seat,
3274                        None,
3275                        Some(format!("the continuation was not started: {e}")),
3276                        ContinuationRecord {
3277                            attempts: attempts.saturating_sub(1),
3278                            cumulative_wait_ms,
3279                            outcome: ContinuationOutcome::NoSession,
3280                        },
3281                    );
3282                }
3283            }
3284        }
3285    }
3286
3287    fn after_implement(&mut self) -> Result<()> {
3288        if !self.state.candidate_preservation.is_empty() {
3289            bail!("candidate preservation incomplete; judging is unsafe");
3290        }
3291        // Scan every candidate patch once the set is complete.
3292        if self.state.leaks.is_empty() {
3293            let cfg = self.state.config.blind.clone();
3294            let mut leaks = Vec::new();
3295            for c in &self.state.candidates {
3296                let Some(patch) =
3297                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3298                else {
3299                    continue;
3300                };
3301                leaks.extend(blind::scan(
3302                    &format!("candidate {} patch", c.label),
3303                    &patch,
3304                    &cfg.vendor_tokens,
3305                ));
3306            }
3307            if !leaks.is_empty() {
3308                let summary = leaks
3309                    .iter()
3310                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3311                    .collect::<Vec<_>>()
3312                    .join(", ");
3313                match cfg.on_leak {
3314                    LeakPolicy::Fail => {
3315                        self.state.status = RunStatus::Failed;
3316                        self.state
3317                            .event("blind", format!("vendor text in a patch: {summary}"));
3318                        self.state.leaks = leaks;
3319                        self.state.save()?;
3320                        self.settle_questions();
3321                        bail!(
3322                            "blind.on_leak = \"fail\" and vendor text reached a \
3323                             judged patch: {summary}"
3324                        );
3325                    }
3326                    LeakPolicy::Redact => self.state.event(
3327                        "blind",
3328                        format!("redacting vendor text for judging: {summary}"),
3329                    ),
3330                    LeakPolicy::Warn => self.state.event(
3331                        "blind",
3332                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3333                    ),
3334                }
3335                self.state.leaks = leaks;
3336            }
3337        }
3338
3339        if self.state.viable().is_empty() {
3340            if self.state.all_candidates_verified_noop() {
3341                // Every candidate agreed, with evidence the adoption guard
3342                // accepted, that nothing belongs in this worktree. That is
3343                // not the same fact as a candidate that simply failed to
3344                // write anything, and settling it as an ordinary `Failed`
3345                // (see `SCHEMA`'s doc for schema 10) is what let two of
3346                // task 391f's attempts burn a retry each re-discovering the
3347                // same already-landed fix. Terminal either way, so `judge`
3348                // must never run over an empty candidate set — unlike the
3349                // `Failed` branch below this returns `Ok`, not an error:
3350                // nothing here failed.
3351                self.state.status = RunStatus::VerifiedNoop;
3352                self.state.save()?;
3353                self.settle_questions();
3354                return Ok(());
3355            }
3356            self.state.status = RunStatus::Failed;
3357            self.state.save()?;
3358            self.settle_questions();
3359            bail!("no candidate produced a change; nothing to judge");
3360        }
3361        self.state.status = RunStatus::Judging;
3362        self.state.save()?;
3363        Ok(())
3364    }
3365
3366    // --------------------------------------------------------------- judge
3367
3368    async fn judge(&mut self) -> Result<()> {
3369        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3370        // agent files with `magi task add` name the run that paid for it. The
3371        // prompt overlay is cloned alongside it because the waves borrow it
3372        // while `self` is mutably borrowed by the node's own bookkeeping.
3373        let run_id = self.state.id.clone();
3374        let prompts = self.state.config.prompts.clone();
3375        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3376            return Ok(());
3377        }
3378        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3379        if viable.len() == 1 {
3380            // Recorded so this is a one-time event: `judgements` stays empty
3381            // either way, which without this flag is indistinguishable from
3382            // "not yet judged" on the next reentry — and status is left
3383            // untouched, so a later node's conclusion (e.g. `Blocked` after
3384            // the review budget ran out) survives a resume instead of being
3385            // clobbered back to `Judging` by this node running again.
3386            self.state.judge_skipped = true;
3387            self.state.event(
3388                "judge",
3389                format!(
3390                    "only candidate {} produced a change; judging skipped",
3391                    viable[0].label
3392                ),
3393            );
3394            self.state.save()?;
3395            return Ok(());
3396        }
3397        self.state.status = RunStatus::Judging;
3398
3399        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3400        let language = self.state.config.graph.language.clone();
3401        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3402        let sessions = self.state.config.graph.sessions;
3403        let artifacts = agent::artifacts_dir(&self.state.dir());
3404        let root = self.state.worktree_root();
3405        let base_short = short(&self.state.base_commit);
3406
3407        let mut jobs = Vec::new();
3408        let mut orders = Vec::new();
3409        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3410            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3411            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3412            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3413            let seat_key = format!("judge-{}", j + 1);
3414            let seat = self.seat(&seat_key, &spec.id);
3415            jobs.push(SeatJob {
3416                prompt: prompt::judge(
3417                    &self.state.instruction,
3418                    &views,
3419                    self.roles.judges.len(),
3420                    &base_short,
3421                    &language,
3422                ),
3423                spec,
3424                seat,
3425                cwd: root.join(format!("judge-{}", j + 1)),
3426                timeout,
3427                allow_write: false,
3428                sessions,
3429                artifacts: artifacts.clone(),
3430                stem: format!("judge-{}", j + 1),
3431                handover: None,
3432            });
3433        }
3434
3435        self.state.event(
3436            "judge",
3437            format!(
3438                "{} judges ranking {} candidates blind",
3439                jobs.len(),
3440                viable.len()
3441            ),
3442        );
3443        let labels_for_check = labels.clone();
3444        let mut quota_losses = Vec::new();
3445        let cache = self.state.config.cache_dir();
3446        let ctx = WaveCtx {
3447            carry_seats: false,
3448            pauses: Some((&self.pause, &self.interrupt)),
3449            run: &run_id,
3450            node: "judge",
3451            prompts: &prompts,
3452            cache: cache.as_deref(),
3453            round: None,
3454        };
3455        let results = ask_json_wave::<Ranking>(
3456            jobs,
3457            Arc::clone(&self.sem),
3458            self.state.config.graph.retries,
3459            &self.roles.judge_roster,
3460            &ctx,
3461            &mut quota_losses,
3462            &mut self.state,
3463            &move |r: &Ranking| r.validate(&labels_for_check),
3464        )
3465        .await;
3466        if self.state.resource_block.is_some() {
3467            return self.stop_for_resource();
3468        }
3469        self.state.quota.extend(quota_losses);
3470
3471        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3472            let agent_id = seat.agent.clone();
3473            self.state.seats.insert(seat.key.clone(), seat);
3474            let mut record = Judgement {
3475                judge: j + 1,
3476                seat: format!("judge-{}", j + 1),
3477                agent: agent_id,
3478                ranking: Vec::new(),
3479                reasons: BTreeMap::new(),
3480                confidence: None,
3481                strengths: BTreeMap::new(),
3482                order: orders[j].clone(),
3483                failed: None,
3484                duration_ms: 0,
3485            };
3486            match res {
3487                Ok((ranking, out)) => {
3488                    record.ranking = ranking.normalized();
3489                    record.reasons = ranking.reasons;
3490                    record.strengths = ranking.strengths;
3491                    record.confidence = ranking.confidence;
3492                    record.duration_ms = out.duration_ms;
3493                    self.state.event(
3494                        "judge",
3495                        format!(
3496                            "judge {} ranked {}",
3497                            j + 1,
3498                            record.ranking.iter().collect::<String>()
3499                        ),
3500                    );
3501                }
3502                Err(e) => {
3503                    record.failed = Some(e.to_string());
3504                    self.state
3505                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3506                }
3507            }
3508            self.state.judgements.push(record);
3509            self.state.save()?;
3510        }
3511        Ok(())
3512    }
3513
3514    // ---------------------------------------------------------- deliberate
3515
3516    async fn deliberate(&mut self) -> Result<()> {
3517        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3518        // agent files with `magi task add` name the run that paid for it. The
3519        // prompt overlay is cloned alongside it because the waves borrow it
3520        // while `self` is mutably borrowed by the node's own bookkeeping.
3521        let run_id = self.state.id.clone();
3522        let prompts = self.state.config.prompts.clone();
3523        if !self.state.deliberation.is_empty() {
3524            return Ok(());
3525        }
3526        let tops: Vec<char> = self
3527            .state
3528            .judgements
3529            .iter()
3530            .filter_map(|j| j.ranking.first().copied())
3531            .collect();
3532        let rounds = self.state.config.graph.deliberate_rounds;
3533        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3534            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3535                self.state.event(
3536                    "deliberate",
3537                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3538                );
3539            }
3540            self.state.status = RunStatus::Voting;
3541            self.state.save()?;
3542            return Ok(());
3543        }
3544
3545        self.state.status = RunStatus::Deliberating;
3546        self.state.event(
3547            "deliberate",
3548            format!(
3549                "split: first choices were {} — opening {rounds} round(s)",
3550                tops.iter().collect::<String>()
3551            ),
3552        );
3553
3554        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3555        let language = self.state.config.graph.language.clone();
3556        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3557        let sessions = self.state.config.graph.sessions;
3558        let artifacts = agent::artifacts_dir(&self.state.dir());
3559        let root = self.state.worktree_root();
3560        let base_short = short(&self.state.base_commit);
3561
3562        // Judges argue in sequence so that a turn can answer the one before it;
3563        // that is the difference between deliberation and three parallel
3564        // monologues.
3565        for round in 1..=rounds {
3566            let mut turns: Vec<DeliberationTurn> = Vec::new();
3567            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3568                if self.state.judgements[j].failed.is_some() {
3569                    continue;
3570                }
3571                let seat_key = format!("judge-{}", j + 1);
3572                let spec = self.occupant(&seat_key, spec);
3573                let mut seat = self.seat(&seat_key, &spec.id);
3574                let transcript = self.transcript(&turns, j);
3575                let build = |context: Option<&str>| {
3576                    prompt::deliberate(
3577                        &self.state.instruction,
3578                        context,
3579                        &transcript,
3580                        round,
3581                        rounds,
3582                        &language,
3583                    )
3584                };
3585                let block = self.candidate_block(&viable, &base_short);
3586                let full = build(Some(&block));
3587                let text = if has_context(&spec, &seat, sessions) {
3588                    build(None)
3589                } else {
3590                    full.clone()
3591                };
3592                let job = SeatJob {
3593                    spec,
3594                    seat: seat.clone(),
3595                    prompt: text,
3596                    cwd: root.join(format!("judge-{}", j + 1)),
3597                    timeout,
3598                    allow_write: false,
3599                    sessions,
3600                    artifacts: artifacts.clone(),
3601                    stem: format!("delib-{round}-judge-{}", j + 1),
3602                    handover: Some(full),
3603                };
3604                let cache = self.state.config.cache_dir();
3605                let ctx = WaveCtx {
3606                    carry_seats: false,
3607                    pauses: Some((&self.pause, &self.interrupt)),
3608                    run: &run_id,
3609                    node: "deliberate",
3610                    prompts: &prompts,
3611                    cache: cache.as_deref(),
3612                    round: None,
3613                };
3614                // A turn is never nudged (`retries` 0): a failed seat is
3615                // handed to the next roster agent, which gets the full
3616                // context. An empty answer is a turn, not a failure.
3617                let mut losses = Vec::new();
3618                let mut results = ask_wave_with::<String>(
3619                    vec![job],
3620                    Arc::clone(&self.sem),
3621                    0,
3622                    &self.roles.judge_roster,
3623                    &ctx,
3624                    &mut losses,
3625                    &mut self.state,
3626                    &|text: &str| {
3627                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3628                    },
3629                )
3630                .await;
3631                self.state.quota.extend(losses);
3632                let (updated, res, _) = results.pop().expect("one job in, one result out");
3633                seat = updated;
3634                let agent_id = seat.agent.clone();
3635                self.state.seats.insert(seat.key.clone(), seat);
3636                let body = match res {
3637                    Ok((body, _)) => body,
3638                    // Skip the seat; a CLI's raw error JSON is never read as
3639                    // this judge's position.
3640                    Err(e) => {
3641                        self.state
3642                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3643                        continue;
3644                    }
3645                };
3646                let tentative = verdict::extract_json::<Position>(&body)
3647                    .ok()
3648                    .and_then(|p| p.tentative)
3649                    .and_then(|s| s.trim().chars().next())
3650                    .map(|c| c.to_ascii_uppercase());
3651                self.state.event(
3652                    "deliberate",
3653                    format!(
3654                        "round {round}: judge {} now favours {}",
3655                        j + 1,
3656                        tentative.map_or("—".to_owned(), |c| c.to_string())
3657                    ),
3658                );
3659                turns.push(DeliberationTurn {
3660                    judge: j + 1,
3661                    agent: agent_id,
3662                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3663                    tentative,
3664                });
3665            }
3666            self.state
3667                .deliberation
3668                .push(DeliberationRound { round, turns });
3669            self.state.save()?;
3670        }
3671
3672        self.state.status = RunStatus::Voting;
3673        self.state.save()?;
3674        Ok(())
3675    }
3676
3677    // ---------------------------------------------------------------- vote
3678
3679    async fn vote(&mut self) -> Result<()> {
3680        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3681        // agent files with `magi task add` name the run that paid for it. The
3682        // prompt overlay is cloned alongside it because the waves borrow it
3683        // while `self` is mutably borrowed by the node's own bookkeeping.
3684        let run_id = self.state.id.clone();
3685        let prompts = self.state.config.prompts.clone();
3686        if !self.state.votes.is_empty() {
3687            return Ok(());
3688        }
3689        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3690        if viable.len() == 1 {
3691            return Ok(());
3692        }
3693        self.state.status = RunStatus::Voting;
3694
3695        let language = self.state.config.graph.language.clone();
3696        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3697        let sessions = self.state.config.graph.sessions;
3698        let artifacts = agent::artifacts_dir(&self.state.dir());
3699        let root = self.state.worktree_root();
3700        let base_short = short(&self.state.base_commit);
3701        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3702
3703        let mut jobs = Vec::new();
3704        let mut seats_at = Vec::new();
3705        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3706            if self
3707                .state
3708                .judgements
3709                .get(j)
3710                .is_some_and(|r| r.failed.is_some())
3711            {
3712                continue;
3713            }
3714            let seat_key = format!("judge-{}", j + 1);
3715            let spec = self.occupant(&seat_key, spec);
3716            let seat = self.seat(&seat_key, &spec.id);
3717            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3718            let text = if has_context(&spec, &seat, sessions) {
3719                prompt::final_vote(&viable, &language)
3720            } else {
3721                full.clone()
3722            };
3723            jobs.push(SeatJob {
3724                spec,
3725                seat,
3726                prompt: text,
3727                cwd: root.join(format!("judge-{}", j + 1)),
3728                timeout,
3729                allow_write: false,
3730                sessions,
3731                artifacts: artifacts.clone(),
3732                stem: format!("vote-judge-{}", j + 1),
3733                handover: Some(full),
3734            });
3735            seats_at.push(j);
3736        }
3737
3738        self.state.event(
3739            "vote",
3740            format!(
3741                "collecting {} final votes one by one, privately",
3742                jobs.len()
3743            ),
3744        );
3745        let allowed = viable.clone();
3746        let mut quota_losses = Vec::new();
3747        let cache = self.state.config.cache_dir();
3748        let ctx = WaveCtx {
3749            carry_seats: false,
3750            pauses: Some((&self.pause, &self.interrupt)),
3751            run: &run_id,
3752            node: "vote",
3753            prompts: &prompts,
3754            cache: cache.as_deref(),
3755            round: None,
3756        };
3757        let results = ask_json_wave::<FinalVote>(
3758            jobs,
3759            Arc::clone(&self.sem),
3760            self.state.config.graph.retries,
3761            &self.roles.judge_roster,
3762            &ctx,
3763            &mut quota_losses,
3764            &mut self.state,
3765            &move |v: &FinalVote| match v.label() {
3766                Some(c) if allowed.contains(&c) => Ok(()),
3767                other => bail!("vote {other:?} is not one of {allowed:?}"),
3768            },
3769        )
3770        .await;
3771        if self.state.resource_block.is_some() {
3772            return self.stop_for_resource();
3773        }
3774        self.state.quota.extend(quota_losses);
3775
3776        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3777            let agent_id = seat.agent.clone();
3778            self.state.seats.insert(seat.key.clone(), seat);
3779            let initial = self
3780                .state
3781                .judgements
3782                .get(j)
3783                .and_then(|r| r.ranking.first().copied());
3784            let mut record = VoteRecord {
3785                judge: j + 1,
3786                agent: agent_id,
3787                vote: None,
3788                reason: String::new(),
3789                changed: false,
3790            };
3791            match res {
3792                Ok((v, _)) => {
3793                    record.vote = v.label();
3794                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3795                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3796                    self.state.event(
3797                        "vote",
3798                        format!(
3799                            "judge {} voted {}{}",
3800                            j + 1,
3801                            record.vote.unwrap_or('?'),
3802                            if record.changed { " (changed)" } else { "" }
3803                        ),
3804                    );
3805                }
3806                Err(e) => {
3807                    self.state
3808                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3809                }
3810            }
3811            self.state.votes.push(record);
3812            self.state.save()?;
3813        }
3814        Ok(())
3815    }
3816
3817    // --------------------------------------------------------------- tally
3818
3819    fn tally(&mut self) -> Result<()> {
3820        if self.state.tally.is_some() {
3821            return Ok(());
3822        }
3823        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3824        let tops: Vec<char> = self
3825            .state
3826            .judgements
3827            .iter()
3828            .filter_map(|j| j.ranking.first().copied())
3829            .collect();
3830        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3831
3832        // A judge whose private vote failed still counted once, in the initial
3833        // ranking; using it beats discarding a whole seat.
3834        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3835        let mut cast: Vec<char> = Vec::new();
3836        for (i, j) in self.state.judgements.iter().enumerate() {
3837            let vote = self
3838                .state
3839                .votes
3840                .iter()
3841                .find(|v| v.judge == i + 1)
3842                .and_then(|v| v.vote)
3843                .or_else(|| j.ranking.first().copied());
3844            if let Some(v) = vote {
3845                *first_choice.entry(v).or_insert(0) += 1;
3846                cast.push(v);
3847            }
3848        }
3849
3850        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3851        for j in &self.state.judgements {
3852            let n = j.ranking.len();
3853            for (pos, label) in j.ranking.iter().enumerate() {
3854                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3855            }
3856        }
3857
3858        let best = first_choice.values().copied().max().unwrap_or(0);
3859        let mut leaders: Vec<char> = first_choice
3860            .iter()
3861            .filter(|(_, v)| **v == best)
3862            .map(|(k, _)| *k)
3863            .collect();
3864        let mut tie_break = None;
3865        if leaders.len() > 1 {
3866            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3867            let borda_leaders: Vec<char> = leaders
3868                .iter()
3869                .copied()
3870                .filter(|l| borda[l] == top_borda)
3871                .collect();
3872            tie_break = Some(if borda_leaders.len() == 1 {
3873                format!(
3874                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3875                    leaders.len()
3876                )
3877            } else {
3878                format!(
3879                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3880                    leaders.len()
3881                )
3882            });
3883            leaders = borda_leaders;
3884            leaders.sort_unstable();
3885        }
3886        let winner = *leaders
3887            .first()
3888            .or(viable.first())
3889            .context("no candidate to declare a winner from")?;
3890
3891        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3892        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3893        let deliberated = !self.state.deliberation.is_empty();
3894
3895        // Whose verdict is this? A rate-limited seat is absent even if it
3896        // ranked before the limit hit, so presence is measured against the
3897        // recorded losses, not just "did a ranking ever appear".
3898        let quota_seats: std::collections::BTreeSet<&str> =
3899            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3900        let mut present = 0usize;
3901        for (i, j) in self.state.judgements.iter().enumerate() {
3902            if quota_seats.contains(j.seat.as_str()) {
3903                continue;
3904            }
3905            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3906            let voted = self
3907                .state
3908                .votes
3909                .iter()
3910                .any(|v| v.judge == i + 1 && v.vote.is_some());
3911            if ranked || voted {
3912                present += 1;
3913            }
3914        }
3915        // Strict majority of the configured panel. A bare majority is real
3916        // signal we can act on, while a minority verdict must never stand in
3917        // for a healthy one. A one-candidate run needs no panel at all, and
3918        // `judges` stays `0` rather than the roster size a panel that never
3919        // sat would otherwise be credited with.
3920        let needs_quorum = viable.len() > 1;
3921        let judges_total = if needs_quorum {
3922            self.roles.judges.len()
3923        } else {
3924            0
3925        };
3926        let quorum = if needs_quorum {
3927            judges_total / 2 + 1
3928        } else {
3929            0
3930        };
3931        let met_quorum = !needs_quorum || present >= quorum;
3932        let uncontested = (!needs_quorum).then(|| {
3933            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3934        });
3935
3936        self.state.event(
3937            "tally",
3938            match &uncontested {
3939                Some(reason) => format!("winner {winner} — {reason}"),
3940                None => format!(
3941                    "winner {winner} — votes {} | initial {} | {} changed | \
3942                     {present}/{judges_total} judges{}",
3943                    first_choice
3944                        .iter()
3945                        .map(|(k, v)| format!("{k}:{v}"))
3946                        .collect::<Vec<_>>()
3947                        .join(" "),
3948                    if unanimous_initial {
3949                        "unanimous"
3950                    } else {
3951                        "split"
3952                    },
3953                    changed_votes,
3954                    if met_quorum {
3955                        String::new()
3956                    } else {
3957                        format!(" — below quorum ({quorum} required)")
3958                    },
3959                ),
3960            },
3961        );
3962        if !met_quorum {
3963            self.state.event(
3964                "stall",
3965                format!(
3966                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3967                     the run stops here, resumable"
3968                ),
3969            );
3970        }
3971        self.state.tally = Some(Tally {
3972            first_choice,
3973            borda,
3974            winner,
3975            rankings: tops.len(),
3976            unanimous_initial,
3977            deliberated,
3978            changed_votes,
3979            unanimous_final,
3980            tie_break,
3981            judges: judges_total,
3982            present,
3983            quorum,
3984            met_quorum,
3985            uncontested,
3986        });
3987        self.state.status = if met_quorum {
3988            RunStatus::Reviewing
3989        } else {
3990            RunStatus::Stalled
3991        };
3992        self.state.save()?;
3993        Ok(())
3994    }
3995
3996    // ------------------------------------------------------------- recover
3997
3998    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3999    /// resumed toward completion once the transient cause clears.
4000    ///
4001    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4002    /// it toward the quorum, which is exactly the set of seats whose absence
4003    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4004    /// not depend on which node happened to hit the limit), or an ordinary
4005    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4006    /// seat is never disturbed.
4007    ///
4008    /// A seat that now answers with a usable ranking is "recovered": its
4009    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4010    /// `tally` counts it present again), and its vote re-collected. A seat that
4011    /// still fails keeps its loss and stays absent.
4012    ///
4013    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4014    /// to review/gate/merge), `false` when it is still below quorum (the run
4015    /// stays `Stalled`, still resumable for a later retry).
4016    #[allow(clippy::too_many_lines)]
4017    async fn recover_stall(&mut self) -> Result<bool> {
4018        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4019        // agent files with `magi task add` name the run that paid for it. The
4020        // prompt overlay is cloned alongside it because the waves borrow it
4021        // while `self` is mutably borrowed by the node's own bookkeeping.
4022        let run_id = self.state.id.clone();
4023        let prompts = self.state.config.prompts.clone();
4024        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4025        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4026        // is what keeps a non-quota collapse — or a quota loss recorded at the
4027        // deliberate node — from being a permanent dead-end on `--resume`.
4028        let quota_seats: BTreeSet<&str> =
4029            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4030        let absent: Vec<String> = self
4031            .state
4032            .judgements
4033            .iter()
4034            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4035            .map(|j| j.seat.clone())
4036            .collect();
4037        if absent.is_empty() {
4038            return Ok(false);
4039        }
4040        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4041        if viable.len() <= 1 {
4042            return Ok(false);
4043        }
4044        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4045        let language = self.state.config.graph.language.clone();
4046        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4047        let sessions = self.state.config.graph.sessions;
4048        let artifacts = agent::artifacts_dir(&self.state.dir());
4049        let root = self.state.worktree_root();
4050        let base_short = short(&self.state.base_commit);
4051        let candidates: Vec<Candidate> = viable.clone();
4052
4053        // Map each absent seat key to its 0-based position in `roles.judges`.
4054        let mut positions: Vec<usize> = absent
4055            .iter()
4056            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4057            .collect();
4058        if positions.is_empty() {
4059            return Ok(false);
4060        }
4061        positions.sort_unstable();
4062        positions.dedup();
4063
4064        // Re-rank the lost seats, one blind prompt each.
4065        let mut judge_jobs = Vec::new();
4066        for &j in &positions {
4067            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4068            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4069            let seat_key = format!("judge-{}", j + 1);
4070            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4071            let seat = self.seat(&seat_key, &spec.id);
4072            judge_jobs.push(SeatJob {
4073                spec,
4074                seat,
4075                prompt: prompt::judge(
4076                    &self.state.instruction,
4077                    &views,
4078                    self.roles.judges.len(),
4079                    &base_short,
4080                    &language,
4081                ),
4082                cwd: root.join(seat_key),
4083                timeout,
4084                allow_write: false,
4085                sessions,
4086                artifacts: artifacts.clone(),
4087                stem: format!("judge-{}-recover", j + 1),
4088                handover: None,
4089            });
4090        }
4091
4092        let labels_for_check = labels.clone();
4093        let mut judge_losses = Vec::new();
4094        let retries = self.state.config.graph.retries;
4095        let cache = self.state.config.cache_dir();
4096        let ctx = WaveCtx {
4097            carry_seats: false,
4098            pauses: Some((&self.pause, &self.interrupt)),
4099            run: &run_id,
4100            node: "judge",
4101            prompts: &prompts,
4102            cache: cache.as_deref(),
4103            round: None,
4104        };
4105        let results = ask_json_wave::<Ranking>(
4106            judge_jobs,
4107            Arc::clone(&self.sem),
4108            retries,
4109            &self.roles.judge_roster,
4110            &ctx,
4111            &mut judge_losses,
4112            &mut self.state,
4113            &move |r: &Ranking| r.validate(&labels_for_check),
4114        )
4115        .await;
4116        if self.state.resource_block.is_some() {
4117            self.stop_for_resource()?;
4118            return Ok(false);
4119        }
4120
4121        // Refresh the judgement of every seat that ranked again.
4122        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4123        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4124            let agent_id = seat.agent.clone();
4125            self.state.seats.insert(seat.key.clone(), seat);
4126            let record = &mut self.state.judgements[j];
4127            match res {
4128                Ok((ranking, out)) => {
4129                    record.agent = agent_id;
4130                    record.ranking = ranking.normalized();
4131                    record.reasons = ranking.reasons;
4132                    record.strengths = ranking.strengths;
4133                    record.confidence = ranking.confidence;
4134                    record.failed = None;
4135                    record.duration_ms = out.duration_ms;
4136                    recovered.insert(j);
4137                    self.state.event(
4138                        "recover",
4139                        format!("judge {} ranked again after the limit", j + 1),
4140                    );
4141                }
4142                Err(e) => {
4143                    self.state
4144                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4145                }
4146            }
4147        }
4148
4149        // Re-ask the votes of the seats that recovered a ranking.
4150        let mut vote_jobs = Vec::new();
4151        let mut vote_pos: Vec<usize> = Vec::new();
4152        for &j in &recovered {
4153            let seat_key = format!("judge-{}", j + 1);
4154            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4155            let seat = self.seat(&seat_key, &spec.id);
4156            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4157            let text = if has_context(&spec, &seat, sessions) {
4158                prompt::final_vote(&labels, &language)
4159            } else {
4160                full.clone()
4161            };
4162            vote_jobs.push(SeatJob {
4163                spec,
4164                seat,
4165                prompt: text,
4166                cwd: root.join(seat_key),
4167                timeout,
4168                allow_write: false,
4169                sessions,
4170                artifacts: artifacts.clone(),
4171                stem: format!("vote-judge-{}-recover", j + 1),
4172                handover: Some(full),
4173            });
4174            vote_pos.push(j);
4175        }
4176        let allowed = labels.clone();
4177        let mut vote_losses = Vec::new();
4178        let vote_retries = self.state.config.graph.retries;
4179        let vote_cache = self.state.config.cache_dir();
4180        let ctx = WaveCtx {
4181            carry_seats: false,
4182            pauses: Some((&self.pause, &self.interrupt)),
4183            run: &run_id,
4184            node: "vote",
4185            prompts: &prompts,
4186            cache: vote_cache.as_deref(),
4187            round: None,
4188        };
4189        let votes = ask_json_wave::<FinalVote>(
4190            vote_jobs,
4191            Arc::clone(&self.sem),
4192            vote_retries,
4193            &self.roles.judge_roster,
4194            &ctx,
4195            &mut vote_losses,
4196            &mut self.state,
4197            &move |v: &FinalVote| match v.label() {
4198                Some(c) if allowed.contains(&c) => Ok(()),
4199                other => bail!("vote {other:?} is not one of {allowed:?}"),
4200            },
4201        )
4202        .await;
4203        if self.state.resource_block.is_some() {
4204            self.stop_for_resource()?;
4205            return Ok(false);
4206        }
4207        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4208            let agent_id = seat.agent.clone();
4209            self.state.seats.insert(seat.key.clone(), seat);
4210            match res {
4211                Ok((v, _)) => {
4212                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4213                        rec.vote = v.label();
4214                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4215                    } else {
4216                        self.state.votes.push(VoteRecord {
4217                            judge: j + 1,
4218                            agent: agent_id,
4219                            vote: v.label(),
4220                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4221                            changed: false,
4222                        });
4223                    }
4224                    self.state.event(
4225                        "recover",
4226                        format!("judge {} voted again after the limit", j + 1),
4227                    );
4228                }
4229                Err(e) => {
4230                    self.state
4231                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4232                }
4233            }
4234        }
4235
4236        // A seat that ranked again is present even if its re-vote failed —
4237        // `tally` falls back to the initial ranking's first choice — so clear
4238        // its quota loss. Seats that still fail keep theirs and stay absent.
4239        let recovered_keys: BTreeSet<String> = recovered
4240            .iter()
4241            .map(|&j| format!("judge-{}", j + 1))
4242            .collect();
4243        self.state
4244            .quota
4245            .retain(|q| !recovered_keys.contains(&q.seat));
4246        // A seat that hit the limit again is a fresh loss, not the old one:
4247        // replace the stale entry so the history stays one-per-seat and the
4248        // daemon can tell this attempt's loss from a previous session's.
4249        for loss in judge_losses.into_iter().chain(vote_losses) {
4250            if recovered_keys.contains(&loss.seat) {
4251                continue;
4252            }
4253            self.state.quota.retain(|q| q.seat != loss.seat);
4254            self.state.quota.push(loss);
4255        }
4256
4257        // Recompute the verdict from the refreshed panel.
4258        self.state.tally = None;
4259        self.tally()?;
4260        Ok(self
4261            .state
4262            .tally
4263            .as_ref()
4264            .map(|t| t.met_quorum)
4265            .unwrap_or(false))
4266    }
4267
4268    // ---------------------------------------------------------------- graft
4269
4270    /// Port the strengths the judges attributed to the losing candidates onto
4271    /// the winner, once, between the tally and the review loop.
4272    ///
4273    /// Best-effort by construction: it returns nothing and fails nothing. A
4274    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4275    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4276    /// the run carries on as if the phase were off. See [`graft`] for the
4277    /// decisions and `AGENTS.md` for the invariants.
4278    ///
4279    /// The one exception is a rollback that could not be confirmed: the
4280    /// rejected commit may still be the winner's tip, so the run is blocked
4281    /// (resumable; the record stays `Running`, which makes a resume retry the
4282    /// restore) instead of reviewing, gating and merging it.
4283    async fn graft(&mut self) -> Result<()> {
4284        let Err(e) = self.graft_inner().await else {
4285            return Ok(());
4286        };
4287        if e.is::<crate::repo_lock::ResourceBlocked>() {
4288            return Err(e);
4289        }
4290        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4291            self.state.status = RunStatus::Blocked;
4292            self.state.event("graft", format!("blocked: {e:#}"));
4293            self.state.save()?;
4294            return Err(e);
4295        }
4296        // An error after the `Running` record was saved may have left the
4297        // fixer's work in the tree: the record must not stay unsettled
4298        // (a later resume would reset past the review fixes), so confirm the
4299        // restore now, or block.
4300        if let Some(rec) = self.state.graft.clone()
4301            && !rec.outcome.settled()
4302        {
4303            if let Some(winner) = self.state.winner().cloned() {
4304                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4305                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4306                        return Err(re);
4307                    }
4308                    self.state.status = RunStatus::Blocked;
4309                    self.state.event("graft", format!("blocked: {re:#}"));
4310                    self.state.save()?;
4311                    return Err(re);
4312                }
4313            }
4314            if let Some(g) = self.state.graft.as_mut() {
4315                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4316            }
4317        }
4318        self.state.event("graft", format!("skipped: {e:#}"));
4319        if let Err(e) = self.state.save() {
4320            tracing::warn!("save after graft: {e:#}");
4321        }
4322        Ok(())
4323    }
4324
4325    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4326    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4327    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4328        let _coordination = self.coordinate(wt).await?;
4329        crate::recovery::restore_repair(wt, pre)
4330            .await
4331            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4332    }
4333
4334    /// A skip is a run event, recorded once however often the node is re-entered.
4335    fn graft_skipped(&mut self, why: &str) {
4336        let message = format!("skipped: {why}");
4337        if !self
4338            .state
4339            .events
4340            .iter()
4341            .any(|e| e.node == "graft" && e.message == message)
4342        {
4343            self.state.event("graft", message);
4344        }
4345    }
4346
4347    async fn graft_inner(&mut self) -> Result<()> {
4348        // A round that was started and never finished (crash, kill): the
4349        // fixer may have left anything in the tree, so put the winner back.
4350        if let Some(rec) = self.state.graft.clone()
4351            && !rec.outcome.settled()
4352        {
4353            // Reviews exist only if the graft ended without its record being
4354            // settled (a swallowed error): the tree has moved on since, and
4355            // a reset to `pre` would discard the review fixes.
4356            if self.state.reviews.is_empty()
4357                && let Some(winner) = self.state.winner().cloned()
4358            {
4359                self.graft_restore(&winner.worktree, &rec.pre).await?;
4360            }
4361            if rec.outcome == GraftOutcome::WaitingForRestore {
4362                self.state.graft = None;
4363                self.state.save()?;
4364            } else {
4365                if let Some(g) = self.state.graft.as_mut() {
4366                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4367                }
4368                self.state.event(
4369                    "graft",
4370                    "rolled back: the previous round was interrupted; the winner is untouched",
4371                );
4372                self.state.save()?;
4373                return Ok(());
4374            }
4375        }
4376        let plan = match graft::plan(&self.state) {
4377            Ok(plan) => plan,
4378            Err(skip) => {
4379                if skip != graft::Skip::AlreadyDone {
4380                    self.graft_skipped(skip.reason());
4381                    self.state.save()?;
4382                }
4383                return Ok(());
4384            }
4385        };
4386        let Some(winner) = self.state.winner().cloned() else {
4387            return Ok(());
4388        };
4389        let wt = winner.worktree.clone();
4390        if !git::is_clean(&wt).await? {
4391            self.graft_skipped("the winner's worktree has uncommitted changes");
4392            self.state.save()?;
4393            return Ok(());
4394        }
4395        let pre = git::rev_parse(&wt, "HEAD").await?;
4396        // Saved before the fixer is called: a stop from here on is repaired
4397        // by the branch above on the next entry.
4398        self.state.graft = Some(GraftRecord {
4399            pre: pre.clone(),
4400            post: None,
4401            lines: None,
4402            limit: None,
4403            agent: None,
4404            outcome: GraftOutcome::Running,
4405            requested: graft::requested(&plan),
4406            adoption: None,
4407            adoption_error: None,
4408        });
4409        self.state.event(
4410            "graft",
4411            format!(
4412                "porting strengths of candidate(s) {} onto {}",
4413                plan.strengths
4414                    .iter()
4415                    .map(|(l, _, _)| l.to_string())
4416                    .collect::<Vec<_>>()
4417                    .join(", "),
4418                plan.winner
4419            ),
4420        );
4421        self.state.save()?;
4422
4423        let fix_prompt = prompt::graft(
4424            &self.state.instruction,
4425            plan.winner,
4426            &plan.strengths,
4427            &self.state.config.graph.language,
4428        );
4429        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4430        let sessions = self.state.config.graph.sessions;
4431        let artifacts = agent::artifacts_dir(&self.state.dir());
4432        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4433        let (_, seat, out) = self
4434            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4435                SeatJob {
4436                    prompt: fix_prompt.clone(),
4437                    spec,
4438                    seat,
4439                    cwd: wt.clone(),
4440                    timeout,
4441                    allow_write: true,
4442                    sessions,
4443                    artifacts: artifacts.clone(),
4444                    stem: "graft".to_owned(),
4445                    handover: None,
4446                }
4447            })
4448            .await;
4449        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4450            // Preserve the need to restore earlier fallback edits without
4451            // confusing an unstarted seat with an interrupted invocation.
4452            self.state.graft.as_mut().expect("graft record").outcome =
4453                GraftOutcome::WaitingForRestore;
4454            return self.stop_for_resource();
4455        }
4456        let mut why: Option<String> = None;
4457        match &out {
4458            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4459                why = Some("the fixer produced nothing usable".to_owned());
4460            }
4461            AgentOutcome::Ok(_) => {}
4462            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4463            AgentOutcome::Quota(o) => {
4464                self.state.quota.push(QuotaLoss {
4465                    seat: seat.key.clone(),
4466                    node: "graft".to_owned(),
4467                    at: Timestamp::now(),
4468                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4469                });
4470                why = Some("rate limited (quota)".to_owned());
4471            }
4472            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4473                why = Some(format!(
4474                    "the fixer failed: {}",
4475                    e.lines()
4476                        .next()
4477                        .unwrap_or("")
4478                        .chars()
4479                        .take(160)
4480                        .collect::<String>()
4481                ));
4482            }
4483        }
4484        let agent_id = seat.agent.clone();
4485        self.state.seats.insert(seat.key.clone(), seat);
4486
4487        let mut post = None;
4488        let mut lines = None;
4489        let _coordination = self.coordinate(&wt).await?;
4490        if why.is_none() {
4491            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4492            if let Ok(r) = &rescued {
4493                self.state.note_withheld("graft", &r.withheld);
4494            }
4495            let head = git::rev_parse(&wt, "HEAD").await?;
4496            // What is measured must be what gets reviewed and gated: a failed
4497            // rescue, or anything still uncommitted (withheld lockfiles
4498            // aside, which are never part of the tree), rejects the graft.
4499            let dirty = match &rescued {
4500                Ok(r) => git::status_porcelain(&wt)
4501                    .await
4502                    .map(|st| {
4503                        st.lines().any(|l| {
4504                            !r.withheld
4505                                .iter()
4506                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4507                        })
4508                    })
4509                    .unwrap_or(true),
4510                Err(_) => true,
4511            };
4512            if dirty {
4513                why = Some("the fixer left changes that could not be committed".to_owned());
4514            } else if head == pre {
4515                why = Some("the fixer changed nothing".to_owned());
4516            } else {
4517                lines = git::changed_lines(&wt, &pre, &head).await;
4518                post = Some(head);
4519            }
4520        }
4521        let (adoption, adoption_error) = match &out {
4522            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4523            _ => (None, None),
4524        };
4525        let requested = graft::requested(&plan);
4526        let outcome = match why {
4527            None => {
4528                self.state.event(
4529                    "graft",
4530                    format!(
4531                        "applied: {}",
4532                        lines.map_or_else(
4533                            || "changed lines unavailable".to_owned(),
4534                            |n| format!("{n} changed lines")
4535                        )
4536                    ),
4537                );
4538                GraftOutcome::Applied
4539            }
4540            Some(why) => {
4541                // Not settled until the restore is confirmed: the record
4542                // stays `Running`, so a resume retries it.
4543                crate::recovery::restore_repair(&wt, &pre)
4544                    .await
4545                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4546                self.state.event(
4547                    "graft",
4548                    format!("rolled back: {why}; the winner is untouched"),
4549                );
4550                GraftOutcome::RolledBack(why)
4551            }
4552        };
4553        self.state.graft = Some(GraftRecord {
4554            pre,
4555            post,
4556            lines,
4557            limit: None,
4558            agent: Some(agent_id),
4559            outcome,
4560            requested,
4561            adoption,
4562            adoption_error,
4563        });
4564        self.state.save()?;
4565        Ok(())
4566    }
4567
4568    // ----------------------------------------------------------------- fold
4569
4570    async fn fold_losers(&mut self) -> Result<()> {
4571        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4572            return Ok(());
4573        };
4574        if !self.state.candidate_preservation.is_empty() {
4575            bail!("candidate preservation incomplete; refusing cleanup");
4576        }
4577        let repo = self.state.repo.clone();
4578        let _coordination = self.coordinate(&repo).await?;
4579        let mut folded = Vec::new();
4580        for i in 0..self.state.candidates.len() {
4581            let c = &self.state.candidates[i];
4582            if c.label == winner || c.folded {
4583                continue;
4584            }
4585            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4586            remove_preserved_worktree(&repo, &wt).await?;
4587            if git::branch_exists(&repo, &branch).await?
4588                && !git::branch_delete(&repo, &branch).await?
4589            {
4590                bail!("candidate branch removal refused");
4591            }
4592            self.state.candidates[i].folded = true;
4593            self.state.save()?;
4594            folded.push(label.to_string());
4595        }
4596        // The judges are finished; their checkouts are pure cost from here.
4597        let root = self.state.worktree_root();
4598        for j in 1..=self.roles.judges.len() {
4599            let wt = root.join(format!("judge-{j}"));
4600            if wt.exists() {
4601                remove_preserved_worktree(&repo, &wt).await?;
4602            }
4603        }
4604        // The design-deliberation stage is finished by the time a tally
4605        // exists — same reasoning as the judges above.
4606        if self.state.config.graph.advise {
4607            for k in 1..=self.state.config.graph.advisors {
4608                let wt = root.join(format!("advisor-{k}"));
4609                if wt.exists() {
4610                    remove_preserved_worktree(&repo, &wt).await?;
4611                }
4612            }
4613        }
4614        if !folded.is_empty() {
4615            self.state
4616                .event("fold", format!("folded candidates {}", folded.join(", ")));
4617            self.state.save()?;
4618        }
4619        Ok(())
4620    }
4621
4622    // ------------------------------------------------------------ base sync
4623
4624    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4625    /// anything verifies it.
4626    ///
4627    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4628    /// read whatever is checked out in the winner's worktree. Left alone that
4629    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4630    /// it when the run *branched* - and a run takes long enough that the base
4631    /// has usually moved by the time it gets here. A gate that ran there
4632    /// answers "green on the commit this run started from", not "green on
4633    /// what is about to land", and the difference showed up three times in
4634    /// one day as a green run whose merge would have reverted a file another
4635    /// pull request had already landed.
4636    ///
4637    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4638    /// `land::Step::Rebase` calls, rather than a second implementation of the
4639    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4640    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4641    /// doc warns about ("two answers to a question nobody notices until a
4642    /// diff is wrong").
4643    ///
4644    /// A conflict is not the end of the road: the standing rebase is handed
4645    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4646    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4647    /// land). Once it finishes, review and the gate run as usual on the
4648    /// rebased tree, which is where a breakage the new base caused is caught
4649    /// by the ordinary gate-fix round. magi resolves nothing itself.
4650    ///
4651    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4652    /// `state.base_sync.attempts`, is how many times the base is *rebased
4653    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4654    /// *conflict* was given to a fixer. When the fixer cannot finish the
4655    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4656    /// what was tried (rounds spent, paths still conflicted) and the branch
4657    /// and worktree stay exactly as they were - untouched, for a person to
4658    /// look at - which is also what makes re-entering this function
4659    /// afterwards a no-op instead of a second attempt at the same wall. A
4660    /// push failure ends the same way.
4661    async fn sync_to_base(&mut self) -> Result<()> {
4662        if self.state.status == RunStatus::AlreadyInBase {
4663            return Ok(());
4664        }
4665        let conflicted = self
4666            .state
4667            .base_sync
4668            .as_ref()
4669            .is_some_and(|s| s.conflict.is_some());
4670        let Some(winner) = self.state.winner().cloned() else {
4671            return Ok(());
4672        };
4673
4674        let repo = self.state.repo.clone();
4675        let remote = self.state.config.merge.remote.clone();
4676        let base_branch = self.state.base_branch.clone();
4677        let tracking = format!("{remote}/{base_branch}");
4678
4679        git::fetch(&repo, &remote, &base_branch).await.ok();
4680        // No network, or the remote never had this branch: the run already
4681        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4682        // otherwise), and one that got this far is not blocked by a fetch
4683        // that fails now. It just has no newer tip to compare against.
4684        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4685            return Ok(());
4686        };
4687
4688        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4689        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4690        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4691
4692        // Before any rebase, and before a recorded conflict is honoured: a
4693        // branch whose change reached the base under other commit ids has
4694        // nothing to rebase and nothing to conflict with, and a run that
4695        // already stopped on that phantom conflict recovers here on resume.
4696        // `behind == 0` with head == tip is a branch the base has since taken
4697        // in whole, whether or not a conflict was ever recorded: the ancestry
4698        // proof must still run (`classify` ignores a head still on the start
4699        // commit).
4700        if (behind > 0 || conflicted || head == tip)
4701            && self
4702                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4703                .await?
4704        {
4705            return Ok(());
4706        }
4707        if conflicted {
4708            return Ok(());
4709        }
4710
4711        if behind == 0 {
4712            // A fixer-finished rebase moves the branch ref before the
4713            // winner's worktree is told (`sync_to_head` below). A run that
4714            // died in between resumes here with `behind == 0` and a tree still
4715            // holding the pre-rebase files, which review and the gate would
4716            // then read. That state is exactly: HEAD moved off the tip the
4717            // rebase started from, yet the tree is still identical to that
4718            // tip. A tree with edits of its own differs from it, so nothing
4719            // is thrown away.
4720            if let Some(from) = self
4721                .state
4722                .rebase_fixes
4723                .iter()
4724                .rev()
4725                .find_map(|r| r.from.clone())
4726                && from != head
4727                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4728                    .await
4729                    .is_ok_and(|o| o.ok())
4730            {
4731                git::sync_to_head(&winner.worktree).await?;
4732            }
4733            // An earlier attempt may have rebased the branch locally and died
4734            // before pushing it (only the fresh-rebase arm below pushes).
4735            // Publish it now, so the plain push at PR time is not refused as
4736            // a non-fast-forward. A run already holding a recorded conflict
4737            // never reaches here; that case is out of scope.
4738            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4739            if let Some(why) = &conflict {
4740                self.state.status = RunStatus::Blocked;
4741                self.state.event("land", why.clone());
4742            }
4743            self.state.base_sync = Some(BaseSync {
4744                tip,
4745                behind: 0,
4746                attempts,
4747                conflict,
4748                already_in: None,
4749            });
4750            self.state.save()?;
4751            return Ok(());
4752        }
4753
4754        if attempts >= BASE_SYNC_ROUNDS {
4755            let why = format!(
4756                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4757                 rebase(s); rebasing again would only race it",
4758                winner.branch
4759            );
4760            self.state.status = RunStatus::Blocked;
4761            self.state.base_sync = Some(BaseSync {
4762                tip,
4763                behind,
4764                attempts,
4765                conflict: Some(why.clone()),
4766                already_in: None,
4767            });
4768            self.state.event("land", why);
4769            self.state.save()?;
4770            return Ok(());
4771        }
4772
4773        self.state.event(
4774            "land",
4775            format!(
4776                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4777                winner.branch
4778            ),
4779        );
4780        self.state.save()?;
4781
4782        // The remote's copy of the branch, read now and only if the fetch
4783        // really succeeded (a stale tracking ref must never pin a lease). It is
4784        // pushed over after a rebase only when it is a commit this branch
4785        // already contains, by ancestry or by patch (an earlier rebase of ours
4786        // that never reached the remote): anything else is somebody else's work.
4787        let branch_tracking = format!("{remote}/{}", winner.branch);
4788        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4789        let remote_tip = if self.state.task_pr_branch().is_none()
4790            && matches!(&fetched_branch, Ok(o) if o.ok())
4791        {
4792            git::rev_parse(&repo, &branch_tracking).await.ok()
4793        } else {
4794            None
4795        };
4796        // A remote tip this branch does not contain is somebody else's work:
4797        // rebasing would leave a local tip that can never be pushed. Stop
4798        // before touching anything and say so.
4799        if let Some(theirs) = &remote_tip
4800            && !git::is_ancestor(&repo, theirs, &head).await
4801            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4802                .await
4803                .is_ok_and(|missing| missing.is_empty())
4804        {
4805            let why = format!(
4806                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4807                 them",
4808                short(theirs),
4809                winner.branch
4810            );
4811            self.state.status = RunStatus::Blocked;
4812            self.state.base_sync = Some(BaseSync {
4813                tip,
4814                behind,
4815                attempts,
4816                conflict: Some(why.clone()),
4817                already_in: None,
4818            });
4819            self.state.event("land", why);
4820            self.state.save()?;
4821            return Ok(());
4822        }
4823
4824        let scratch = self.state.dir().join("base-sync");
4825        let rebased = match crate::rebase::rebase_with_fixer(
4826            &mut self.state,
4827            &scratch,
4828            &winner.branch,
4829            &tracking,
4830        )
4831        .await
4832        {
4833            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4834            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4835            Err(e) => Err(e),
4836        };
4837        let attempts = attempts + 1;
4838        match rebased {
4839            Ok(None) => {
4840                // The branch ref moved, but a worktree that already had it
4841                // checked out (the winner's) was not told; sync its index and
4842                // files before anything reads them.
4843                git::sync_to_head(&winner.worktree).await?;
4844                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4845                let mut conflict = None;
4846                if let Some(pinned) = &remote_tip {
4847                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4848                    match pushed {
4849                        Ok(o) if o.ok() => self.state.event(
4850                            "land",
4851                            format!("pushed rebased {} to {remote}", winner.branch),
4852                        ),
4853                        Ok(o) => {
4854                            conflict = Some(format!(
4855                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4856                                winner.branch,
4857                                short(pinned),
4858                                o.stderr.chars().take(600).collect::<String>()
4859                            ));
4860                        }
4861                        Err(e) => {
4862                            conflict = Some(format!(
4863                                "rebased {} locally but could not push it: {e:#}",
4864                                winner.branch
4865                            ));
4866                        }
4867                    }
4868                }
4869                if let Some(why) = &conflict {
4870                    self.state.status = RunStatus::Blocked;
4871                    self.state.event("land", why.clone());
4872                }
4873                self.state.base_sync = Some(BaseSync {
4874                    tip: tip.clone(),
4875                    behind: 0,
4876                    attempts,
4877                    conflict,
4878                    already_in: None,
4879                });
4880                self.state
4881                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4882            }
4883            Ok(Some(conflict)) => {
4884                let why = format!(
4885                    "{} conflicts with {tracking} and did not rebase: {}",
4886                    winner.branch,
4887                    conflict.chars().take(600).collect::<String>()
4888                );
4889                self.state.status = RunStatus::Blocked;
4890                self.state.base_sync = Some(BaseSync {
4891                    tip,
4892                    behind,
4893                    attempts,
4894                    conflict: Some(why.clone()),
4895                    already_in: None,
4896                });
4897                self.state.event("land", why);
4898            }
4899            // A refused Git write or a busy repository: nothing is wrong with
4900            // the branch. The scratch rebase stays standing and the run
4901            // resumes into it (`base_sync` must not claim a conflict).
4902            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4903                self.state.save()?;
4904                return Ok(());
4905            }
4906            Err(e) => {
4907                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4908                self.state.status = RunStatus::Blocked;
4909                self.state.base_sync = Some(BaseSync {
4910                    tip,
4911                    behind,
4912                    attempts,
4913                    conflict: Some(why.clone()),
4914                    already_in: None,
4915                });
4916                self.state.event("land", why);
4917            }
4918        }
4919        self.state.save()?;
4920        Ok(())
4921    }
4922
4923    /// Push a branch an earlier attempt rebased locally but never published,
4924    /// pinned to the remote tip read right after a successful fetch. Returns
4925    /// the reason when the run must stop; `None` when there was nothing to do
4926    /// (no remote copy, the same tip, or a remote copy this branch already
4927    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4928    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4929        // Task competitions publish only after the PR text gate, to their
4930        // public head; never resurrect a candidate branch on the remote.
4931        if self.state.task_pr_branch().is_some() {
4932            return None;
4933        }
4934        let repo = self.state.repo.clone();
4935        let remote = self.state.config.merge.remote.clone();
4936        let fetched = git::fetch(&repo, &remote, branch).await;
4937        if !matches!(&fetched, Ok(o) if o.ok()) {
4938            return None;
4939        }
4940        let branch_tracking = format!("{remote}/{branch}");
4941        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4942        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4943            return None;
4944        }
4945        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4946            .await
4947            .is_ok_and(|missing| missing.is_empty())
4948        {
4949            return Some(format!(
4950                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4951                 them",
4952                short(&theirs)
4953            ));
4954        }
4955        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4956            Ok(o) if o.ok() => {
4957                self.state
4958                    .event("land", format!("pushed rebased {branch} to {remote}"));
4959                None
4960            }
4961            Ok(o) => Some(format!(
4962                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4963                 someone may have pushed): {}",
4964                short(&theirs),
4965                o.stderr.chars().take(600).collect::<String>()
4966            )),
4967            Err(e) => Some(format!(
4968                "{branch} is rebased locally but could not be pushed: {e:#}"
4969            )),
4970        }
4971    }
4972
4973    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4974    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4975    /// whether it did.
4976    ///
4977    /// Checked only when the base is ahead of the branch. A failing check is
4978    /// "not proven" - the ordinary rebase path then decides - never a reason to
4979    /// stop the run.
4980    ///
4981    /// The remote copy of the branch is held to the same standard as the local
4982    /// one: if it carries a tip this worktree does not, that tip must itself be
4983    /// proven in the base, or nothing is settled (a pull request would
4984    /// otherwise be closed over commits nobody checked). The pull request is
4985    /// closed *before* the terminal status is saved; if that fails for a
4986    /// reason other than a refusal (no network, a `gh` error) the run is left
4987    /// `Blocked` with the reason as its conflict, which a resume retries -
4988    /// the same recovery a phantom conflict gets.
4989    async fn settle_already_in(
4990        &mut self,
4991        branch: &str,
4992        tip: &str,
4993        head: &str,
4994        attempts: usize,
4995        behind: usize,
4996    ) -> Result<bool> {
4997        let repo = self.state.repo.clone();
4998        let remote = self.state.config.merge.remote.clone();
4999        let start = self.state.base_commit.clone();
5000        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5001            Ok(Some(e)) => e,
5002            Ok(None) => return Ok(false),
5003            Err(e) => {
5004                tracing::warn!("already-in-base check for {branch}: {e:#}");
5005                return Ok(false);
5006            }
5007        };
5008        let mut verified = vec![head.to_owned()];
5009        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5010        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5011        if matches!(&fetched, Ok(o) if o.ok())
5012            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5013            && theirs != head
5014        {
5015            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5016                Ok(Some(_)) => verified.push(theirs),
5017                _ => return Ok(false),
5018            }
5019        }
5020        let base_branch = self.state.base_branch.clone();
5021        let message = format!(
5022            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5023             land",
5024            evidence.names(),
5025            evidence.proof.as_str()
5026        );
5027        let closed =
5028            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5029                .await;
5030        match closed {
5031            Ok(Ok(url)) => self
5032                .state
5033                .event("land", format!("closed {url}: superseded on {base_branch}")),
5034            Ok(Err(why)) => self
5035                .state
5036                .event("land", format!("did not close a pull request: {why}")),
5037            Err(e) => {
5038                let why = format!(
5039                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5040                     not be closed ({e:#}); resume to retry"
5041                );
5042                self.state.status = RunStatus::Blocked;
5043                self.state.base_sync = Some(BaseSync {
5044                    tip: tip.to_owned(),
5045                    behind,
5046                    attempts,
5047                    conflict: Some(why.clone()),
5048                    already_in: None,
5049                });
5050                self.state.event("land", why);
5051                self.state.save()?;
5052                return Ok(true);
5053            }
5054        }
5055        self.state.status = RunStatus::AlreadyInBase;
5056        self.state.base_sync = Some(BaseSync {
5057            tip: tip.to_owned(),
5058            behind,
5059            attempts,
5060            conflict: None,
5061            already_in: Some(evidence),
5062        });
5063        self.state.event("land", message);
5064        self.state.save()?;
5065        self.settle_questions();
5066        Ok(true)
5067    }
5068
5069    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5070    /// last landed the winner on, once it has run, else the commit the run
5071    /// branched from.
5072    ///
5073    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5074    /// and `vote` all happen before there is a winner to rebase, so they
5075    /// compare every candidate against the branch point on purpose, and a
5076    /// base that moves after they are already done cannot change an answer
5077    /// they already gave.
5078    fn landing_base(&self) -> String {
5079        self.state
5080            .base_sync
5081            .as_ref()
5082            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5083    }
5084
5085    // ------------------------------------------------------- operator fix
5086
5087    /// Route specific, already-recorded review findings to a fixer for a
5088    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5089    /// entry point.
5090    ///
5091    /// Distinct from `review_loop`'s own fix step in three ways: it never
5092    /// runs a reviewer wave, it never spends review-round budget, and what
5093    /// happened is recorded as an [`OperatorFixRequest`] appended to
5094    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5095    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5096    /// and vote must never be rewritten to look like a manufactured blocking
5097    /// verdict.
5098    ///
5099    /// Only meaningful once review has actually concluded: `Ready` (handed
5100    /// off with findings still open, or simply concluded clean while minor
5101    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5102    /// gate failed). Everything else is refused: a run still in progress
5103    /// should simply be resumed, and a `Merged` run's branch has already
5104    /// landed — reopening *this* run's own record cannot change that, so the
5105    /// answer there is a fresh `magi review <branch>`.
5106    ///
5107    /// A real commit here re-verifies through a fresh, ordinary review-only
5108    /// run on the same branch ([`Self::review`]) rather than reopening this
5109    /// run's own `review_loop`: once any round in this run's history went
5110    /// clean, `review_conclusion` treats that as permanent by design (the
5111    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5112    /// way to force one more genuine reviewer wave out of *this* run without
5113    /// either rewriting history or weakening that guarantee for every other
5114    /// caller. A review-only run costs nothing extra — no implementation, no
5115    /// judging, no vote — and exercises the exact same review → verify →
5116    /// gate → (human) merge path, unmodified.
5117    pub async fn fix_selected(
5118        &mut self,
5119        ids: &[String],
5120        reason: &str,
5121        allow_stale: bool,
5122    ) -> Result<()> {
5123        let reason = reason.trim();
5124        if reason.is_empty() {
5125            bail!("a fix request needs a reason — that is the operator's own record of why");
5126        }
5127        if ids.is_empty() {
5128            bail!("no finding id given");
5129        }
5130        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5131            bail!(
5132                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5133                 has already concluded — can be given a targeted fix. A run still \
5134                 in progress should simply be resumed; a `merged` run's branch has \
5135                 already landed, so its answer is a fresh `magi review <branch>`, \
5136                 not reopening this run's own record",
5137                self.state.id,
5138                self.state.status.as_str()
5139            );
5140        }
5141        let Some(winner) = self.state.winner().cloned() else {
5142            bail!("run {} has no winning candidate to fix", self.state.id);
5143        };
5144        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5145            bail!(
5146                "branch `{}` no longer exists; this run cannot be extended",
5147                winner.branch
5148            );
5149        }
5150        let home = crate::run::home();
5151        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5152            bail!(
5153                "run {} is currently being worked on by another magi process",
5154                self.state.id
5155            );
5156        }
5157        // Held for the rest of this call, including the follow-up review
5158        // below: two `magi fix` invocations against the same run must not
5159        // both reach the worktree manipulation further down, which would
5160        // otherwise race to remove and recreate the same directory — see
5161        // [`FixClaim`]'s own doc.
5162        let _claim = FixClaim::acquire(&self.state.dir())?;
5163
5164        // Resolve every id before spending anything — an unknown id refuses
5165        // the whole request rather than silently dropping it — and dedup
5166        // while keeping the operator's own order.
5167        let mut seen = BTreeSet::new();
5168        let mut findings = Vec::new();
5169        let mut missing = Vec::new();
5170        for id in ids {
5171            if !seen.insert(id.clone()) {
5172                continue;
5173            }
5174            match self.state.finding(id) {
5175                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5176                    id: f.id.clone(),
5177                    severity: f.severity,
5178                    reviewer_vote: rec.vote,
5179                    round: round.round,
5180                    round_head: round.head.clone(),
5181                    reviewer: rec.reviewer,
5182                    agent: rec.agent.clone(),
5183                    file: f.file.clone(),
5184                    line: f.line,
5185                    title: f.title.clone(),
5186                    detail: f.detail.clone(),
5187                    outcome: OperatorFixOutcome::Pending,
5188                }),
5189                None => missing.push(id.clone()),
5190            }
5191        }
5192        if !missing.is_empty() {
5193            bail!(
5194                "unknown finding id(s): {}; nothing was changed",
5195                missing.join(", ")
5196            );
5197        }
5198
5199        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5200        let stale_details: Vec<(String, String)> = findings
5201            .iter()
5202            .filter(|f| f.round_head != head_at_request)
5203            .map(|f| (f.id.clone(), f.round_head.clone()))
5204            .collect();
5205        let stale = !stale_details.is_empty();
5206        if stale && !allow_stale {
5207            bail!(
5208                "the branch has moved since some finding(s) were raised — {} — now \
5209                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5210                stale_details
5211                    .iter()
5212                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5213                    .collect::<Vec<_>>()
5214                    .join(", "),
5215                short(&head_at_request)
5216            );
5217        }
5218
5219        let request = OperatorFixRequest {
5220            requested_at: Timestamp::now(),
5221            reason: reason.to_owned(),
5222            findings,
5223            head_at_request: head_at_request.clone(),
5224            allow_stale,
5225            stale,
5226            fix: None,
5227            result_head: None,
5228            follow_up_review_run: None,
5229        };
5230        self.state.event(
5231            "fix",
5232            format!(
5233                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5234                request.findings.len(),
5235                request
5236                    .findings
5237                    .iter()
5238                    .map(|f| f.id.as_str())
5239                    .collect::<Vec<_>>()
5240                    .join(", "),
5241            ),
5242        );
5243        // Recorded now, before any worktree work or the fixer call itself —
5244        // and re-saved at each checkpoint below: a crash at any point after
5245        // this (mid fixer call, mid follow-up review) must not lose the fact
5246        // that this was requested, for which findings, and why. Everything
5247        // past this point reads and writes through `request_index` rather
5248        // than a local variable, since `request` itself is moved here.
5249        self.state.operator_fixes.push(request);
5250        self.state.save()?;
5251        let request_index = self.state.operator_fixes.len() - 1;
5252
5253        // A fresh, dedicated worktree for this one call, never the winner's
5254        // own worktree in place: that one may already be gone (folded away),
5255        // and reusing it in place would leave the branch checked out there
5256        // when the follow-up review below tries to check it out again. Freed
5257        // immediately after, either way — but only once confirmed clean:
5258        // `worktree_remove` is a `git worktree remove --force`, which would
5259        // otherwise discard uncommitted work left there by the operator or
5260        // another process before this had a chance to even look at it.
5261        let repo = self.state.repo.clone();
5262        let coordination = self.coordinate(&repo).await?;
5263        if winner.worktree.exists() {
5264            // Lockfiles a rescue commit withheld stay untracked on purpose and
5265            // are already recorded; they are not the operator's work to protect.
5266            let dirty = git::git(
5267                &winner.worktree,
5268                &["status", "--porcelain", "--untracked-files=all"],
5269            )
5270            .await?;
5271            let only_withheld = dirty.lines().all(|l| {
5272                l.strip_prefix("?? ")
5273                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5274            });
5275            if !only_withheld {
5276                bail!(
5277                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5278                     discard them first",
5279                    winner.worktree.display()
5280                );
5281            }
5282            git::worktree_remove(&self.state.repo, &winner.worktree)
5283                .await
5284                .ok();
5285        }
5286        let fix_worktree = self.state.worktree_root().join("operator-fix");
5287        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5288        git::git(
5289            &self.state.repo,
5290            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5291        )
5292        .await
5293        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5294        if !git::is_clean(&fix_worktree).await? {
5295            git::worktree_remove(&self.state.repo, &fix_worktree)
5296                .await
5297                .ok();
5298            bail!(
5299                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5300                winner.branch
5301            );
5302        }
5303        let repo_for_setup = self.state.repo.clone();
5304        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5305            Ok(()) => {
5306                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5307            }
5308            Err(e) => Err(e),
5309        };
5310        if let Err(e) = setup {
5311            git::worktree_remove(&self.state.repo, &fix_worktree)
5312                .await
5313                .ok();
5314            return Err(e);
5315        }
5316
5317        let run_id = self.state.id.clone();
5318        let prompts = self.state.config.prompts.clone();
5319        let language = self.state.config.graph.language.clone();
5320        let sessions = self.state.config.graph.sessions;
5321        let artifacts = agent::artifacts_dir(&self.state.dir());
5322        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5323            .findings
5324            .iter()
5325            .map(|f| Finding {
5326                id: f.id.clone(),
5327                severity: f.severity,
5328                file: f.file.clone(),
5329                line: f.line,
5330                title: f.title.clone(),
5331                detail: f.detail.clone(),
5332            })
5333            .collect();
5334        let fix_prompt = prompt::operator_fix_in(
5335            &self.state.instruction,
5336            &finding_list,
5337            reason,
5338            &stale_details,
5339            &head_at_request,
5340            &language,
5341            &self.state.config.graph.github_language,
5342        );
5343        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5344        drop(coordination);
5345        let (job, seat, out) = self
5346            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5347                prompt: fix_prompt.clone(),
5348                spec,
5349                seat,
5350                cwd: fix_worktree.clone(),
5351                timeout,
5352                allow_write: true,
5353                sessions,
5354                artifacts: artifacts.clone(),
5355                stem: "operator-fix".to_owned(),
5356                handover: None,
5357            })
5358            .await;
5359        let agent_id = seat.agent.clone();
5360
5361        let mut fix = FixRecord {
5362            agent: agent_id,
5363            addressed: Vec::new(),
5364            rejected: Vec::new(),
5365            notes: String::new(),
5366            committed: false,
5367            failed: None,
5368            duration_ms: 0,
5369            continuation: None,
5370        };
5371        let mut final_seat = seat.clone();
5372        match out {
5373            AgentOutcome::Ok(o) => {
5374                fix.duration_ms = o.duration_ms;
5375                let parsed = verdict::extract_json::<FixReport>(&o.text);
5376                let incomplete_reason = match &parsed {
5377                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5378                        "the reply parsed, but it reported a command whose own CLI \
5379                         never confirmed an exit status"
5380                            .to_owned(),
5381                    ),
5382                    Ok(_) => None,
5383                    Err(e) => Some(e.to_string()),
5384                };
5385                match incomplete_reason {
5386                    None => {
5387                        let report = parsed.expect("checked Ok above");
5388                        fix.addressed = report.addressed;
5389                        fix.rejected = report.rejected;
5390                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5391                    }
5392                    Some(reason) => {
5393                        let (resumed_seat, resolved, failure, cont) = self
5394                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5395                            .await;
5396                        fix.duration_ms += cont.cumulative_wait_ms;
5397                        fix.continuation = Some(cont);
5398                        final_seat = resumed_seat;
5399                        match resolved {
5400                            Some(report) => {
5401                                fix.addressed = report.addressed;
5402                                fix.rejected = report.rejected;
5403                                fix.notes =
5404                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5405                            }
5406                            None => fix.failed = failure,
5407                        }
5408                    }
5409                }
5410            }
5411            AgentOutcome::Dropped(o) => {
5412                fix.duration_ms = o.duration_ms;
5413                let why = o
5414                    .dropped
5415                    .as_ref()
5416                    .map(|d| d.why.as_str())
5417                    .unwrap_or("the CLI ended the stream without delivering its answer");
5418                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5419            }
5420            AgentOutcome::Quota(o) => {
5421                self.state.quota.push(QuotaLoss {
5422                    seat: final_seat.key.clone(),
5423                    node: "fix".to_owned(),
5424                    at: Timestamp::now(),
5425                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5426                });
5427                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5428            }
5429            AgentOutcome::Failed(e) => fix.failed = Some(e),
5430            AgentOutcome::ResourceBlocked(e) => {
5431                fix.failed = Some(format!("the fixer was not started: {e}"));
5432            }
5433        }
5434        if fix.continuation.is_none() {
5435            fix.continuation = Some(ContinuationRecord::not_needed());
5436        }
5437        self.state.seats.insert(final_seat.key.clone(), final_seat);
5438
5439        let rescue_message = format!(
5440            "magi: operator-selected fix ({}) (uncommitted work)",
5441            self.state.operator_fixes[request_index]
5442                .findings
5443                .iter()
5444                .map(|f| f.id.as_str())
5445                .collect::<Vec<_>>()
5446                .join(", ")
5447        );
5448        let repo = self.state.repo.clone();
5449        let coordination = self.coordinate(&repo).await?;
5450        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5451            self.state.note_withheld("fix", &r.withheld);
5452        }
5453        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5454        fix.committed = after != head_at_request;
5455        git::worktree_remove(&self.state.repo, &fix_worktree)
5456            .await
5457            .ok();
5458        drop(coordination);
5459
5460        self.state.event(
5461            "fix",
5462            match &fix.failed {
5463                Some(reason) => format!(
5464                    "operator fix: adoption report was lost ({reason}); {}",
5465                    if fix.committed {
5466                        "committed"
5467                    } else {
5468                        "NO new commit"
5469                    }
5470                ),
5471                None => format!(
5472                    "operator fix: {} addressed, {} rejected, {}",
5473                    fix.addressed.len(),
5474                    fix.rejected.len(),
5475                    if fix.committed {
5476                        "committed"
5477                    } else {
5478                        "NO new commit"
5479                    }
5480                ),
5481            },
5482        );
5483
5484        // Every selected finding gets an outcome — never left `Pending` once
5485        // the fixer's own turn is over. A report that never came back at all
5486        // marks every one of them `Unreported`, not silently "not addressed":
5487        // quota, a dropped stream, or an exhausted continuation are gaps in
5488        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5489        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5490        for f in &mut self.state.operator_fixes[request_index].findings {
5491            f.outcome = if fix.failed.is_some() {
5492                OperatorFixOutcome::Unreported
5493            } else if fix.addressed.contains(&f.id) {
5494                OperatorFixOutcome::Addressed
5495            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5496                OperatorFixOutcome::Rejected { why: r.why.clone() }
5497            } else {
5498                OperatorFixOutcome::Unreported
5499            };
5500        }
5501
5502        let committed = fix.committed;
5503        if committed {
5504            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5505        }
5506        self.state.operator_fixes[request_index].fix = Some(fix);
5507        // Saved again now that the fixer's own outcome is final, on top of
5508        // the save right after the request was first pushed above.
5509        self.state.save()?;
5510
5511        if committed {
5512            self.state.event(
5513                "fix",
5514                format!(
5515                    "operator fix committed {}; opening a follow-up review-only run",
5516                    short(&after)
5517                ),
5518            );
5519            // The operator asked for the fix, and the follow-up serves whatever
5520            // task the run it follows served.
5521            let origin =
5522                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5523            match Self::review(
5524                &self.state.repo,
5525                &winner.branch,
5526                self.state.config.clone(),
5527                origin,
5528            )
5529            .await
5530            {
5531                Ok(mut follow_up) => {
5532                    follow_up.state.event(
5533                        "start",
5534                        format!(
5535                            "requested by an operator fix on run {} for finding(s) {}",
5536                            self.state.id,
5537                            self.state.operator_fixes[request_index]
5538                                .findings
5539                                .iter()
5540                                .map(|f| f.id.as_str())
5541                                .collect::<Vec<_>>()
5542                                .join(", "),
5543                        ),
5544                    );
5545                    follow_up.state.save()?;
5546                    let follow_up_id = follow_up.state.id.clone();
5547                    // The follow-up is a run like any other: it belongs to the
5548                    // task of the run it follows, or to one filed for it.
5549                    let adopted = match crate::direct::adopt(
5550                        &follow_up.state,
5551                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5552                    ) {
5553                        Ok(a) => a,
5554                        Err(e) => {
5555                            // An ownerless run must not spend agent calls; it
5556                            // stays saved, and `magi run --resume` adopts it.
5557                            self.state.event(
5558                                "fix",
5559                                format!(
5560                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5561                                ),
5562                            );
5563                            self.state.save()?;
5564                            return Ok(());
5565                        }
5566                    };
5567                    let executed = follow_up.execute().await;
5568                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5569                    if let Some(a) = adopted {
5570                        a.finish(&follow_up.state, executed);
5571                    }
5572                    if let Some(e) = failure {
5573                        self.state.event(
5574                            "fix",
5575                            format!(
5576                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5577                            ),
5578                        );
5579                    }
5580                    self.state.operator_fixes[request_index].follow_up_review_run =
5581                        Some(follow_up_id);
5582                }
5583                Err(e) => {
5584                    self.state.event(
5585                        "fix",
5586                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5587                    );
5588                }
5589            }
5590            self.state.save()?;
5591        }
5592
5593        Ok(())
5594    }
5595
5596    // --------------------------------------------------------------- review
5597
5598    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5599    /// names, in order, each at most once, moving on only when the call
5600    /// advances (an error, a quota hit or nothing usable - the same decision
5601    /// point `agent::chain_advances` is for the other chained roles).
5602    ///
5603    /// Each agent gets its own seat from `Runner::seat`: the same agent
5604    /// continues its conversation (the winner's own implementer seat when it
5605    /// is the winner's author, now that the competition is over), another
5606    /// takes a fresh one so the full prompt is sent again. A handover is
5607    /// recorded under `node`, which is what makes the fallback stick for the
5608    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5609    /// is returned, so an exhausted chain reads exactly like a single failed
5610    /// fixer: one quota loss, the same wording, the same refund. An earlier
5611    /// attempt's edits are left in the tree and judged, with the final
5612    /// attempt's, by what git says afterwards.
5613    ///
5614    /// Returns the job that produced the outcome, for `continue_fix_report`.
5615    async fn ask_fixer(
5616        &mut self,
5617        winner: &Candidate,
5618        node: &'static str,
5619        round: Option<usize>,
5620        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5621    ) -> (SeatJob, SeatState, AgentOutcome) {
5622        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5623        self.ask_fixer_among(attempts, node, round, None, build)
5624            .await
5625    }
5626
5627    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5628    /// worktree and a commit: before the next agent is asked after one that
5629    /// did not satisfy the call, the worktree is put back on that commit, so
5630    /// a failed agent's partial edits are not carried into the next one.
5631    async fn ask_fixer_among(
5632        &mut self,
5633        attempts: Vec<(AgentSpec, String)>,
5634        node: &'static str,
5635        round: Option<usize>,
5636        restore: Option<(&Path, &str)>,
5637        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5638    ) -> (SeatJob, SeatState, AgentOutcome) {
5639        let run_id = self.state.id.clone();
5640        let prompts = self.state.config.prompts.clone();
5641        let cache = self.state.config.cache_dir();
5642        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5643        let mut last = None;
5644        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5645            let seat = self.seat(&key, &spec.id);
5646            let mut job = build(spec.clone(), seat);
5647            if i > 0 {
5648                job.stem = format!("{}-{}", job.stem, spec.id);
5649            }
5650            let ctx = WaveCtx {
5651                carry_seats: false,
5652                pauses: Some((&self.pause, &self.interrupt)),
5653                run: &run_id,
5654                node,
5655                prompts: &prompts,
5656                cache: cache.as_deref(),
5657                round,
5658            };
5659            let (seat, out) =
5660                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5661            // A resource block never advances the chain: the chosen agent
5662            // was not asked, so it must be the one asked again on resume.
5663            let advances = match &out {
5664                AgentOutcome::Ok(o) => agent::output_advances(o),
5665                AgentOutcome::ResourceBlocked(_) => false,
5666                _ => true,
5667            };
5668            if let Some(next) = ids.get(i + 1)
5669                && advances
5670            {
5671                self.state.seats.insert(seat.key.clone(), seat.clone());
5672                let class =
5673                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5674                let reason = match &out {
5675                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5676                    other => fail_reason(other),
5677                };
5678                // A failed agent's edits must not reach the next one, and a
5679                // restore that did not work ends the chain: the call fails
5680                // and the caller's own (confirmed) restore decides.
5681                if let Some((wt, rev)) = restore
5682                    && let Err(e) = self.graft_restore(wt, rev).await
5683                {
5684                    let why =
5685                        format!("could not restore the worktree before the next fixer: {e:#}");
5686                    tracing::warn!("{why}");
5687                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5688                        AgentOutcome::ResourceBlocked(why)
5689                    } else {
5690                        AgentOutcome::Failed(why)
5691                    };
5692                    last = Some((job, seat, out));
5693                    break;
5694                }
5695                record_handover(
5696                    &mut self.state,
5697                    node,
5698                    &seat.key,
5699                    &spec.id,
5700                    next,
5701                    &class,
5702                    &reason,
5703                );
5704                continue;
5705            }
5706            last = Some((job, seat, out));
5707            break;
5708        }
5709        last.expect("the fixer chain always has an entry")
5710    }
5711
5712    /// The one repair a verification-only failure earns (see
5713    /// [`crate::recovery`]), before the first review round of a run that took
5714    /// a branch over.
5715    ///
5716    /// Saved as started *before* the fixer is called, so a resume never
5717    /// repairs twice; a start found on re-entry is rolled back to the commit
5718    /// it began from. A failed or unusable fixer leaves the tree as it was and
5719    /// the review goes on over it, so the first round's own verification
5720    /// reports the failure again. The failure is never carried into a round as
5721    /// a result: verification is re-run on whatever commit this leaves.
5722    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5723        use crate::recovery::{Mode, Repair};
5724        if !self.state.reviews.is_empty() {
5725            return Ok(());
5726        }
5727        let Some(mut rec) = self.state.recovery.clone() else {
5728            return Ok(());
5729        };
5730        if rec.mode != Mode::VerifyRepair {
5731            return Ok(());
5732        }
5733        let wt = winner.worktree.clone();
5734        let mut head = git::rev_parse(&wt, "HEAD").await?;
5735        match rec.repair.clone() {
5736            Repair::Done { .. } => return Ok(()),
5737            Repair::WaitingForRestore { before } => {
5738                self.graft_restore(&wt, &before).await?;
5739                head = before;
5740                rec.repair = Repair::NotStarted;
5741                self.state.recovery = Some(rec.clone());
5742                self.state.save()?;
5743            }
5744            Repair::Started { before } => {
5745                // Interrupted: the unfinished edits are not trusted.
5746                // An unconfirmed rollback blocks the run with `Started` kept,
5747                // so a resume retries it; the tree is never reviewed as is.
5748                if let Err(e) = self.graft_restore(&wt, &before).await {
5749                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5750                        return Err(e);
5751                    }
5752                    return self
5753                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5754                        .await;
5755                }
5756                rec.repair = Repair::Done {
5757                    before: before.clone(),
5758                    after: before,
5759                    committed: false,
5760                    failed: Some("interrupted; rolled back".to_owned()),
5761                };
5762                self.state
5763                    .event("recovery", "the repair was interrupted and rolled back");
5764                self.state.recovery = Some(rec);
5765                self.state.save()?;
5766                return Ok(());
5767            }
5768            Repair::NotStarted => {}
5769        }
5770        if crate::recovery::recheck(&mut rec, &head) {
5771            self.state.event("recovery", rec.summary());
5772            self.state.recovery = Some(rec);
5773            self.state.save()?;
5774            return Ok(());
5775        }
5776        rec.repair = Repair::Started {
5777            before: head.clone(),
5778        };
5779        self.state.recovery = Some(rec.clone());
5780        self.state.save()?;
5781
5782        let language = self.state.config.graph.language.clone();
5783        let sessions = self.state.config.graph.sessions;
5784        let artifacts = agent::artifacts_dir(&self.state.dir());
5785        let fix_prompt = prompt::recovery_fix(
5786            &self.state.instruction,
5787            &rec,
5788            self.state.config.graph.review_rounds,
5789            &language,
5790        );
5791        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5792        self.state.event(
5793            "recovery",
5794            format!(
5795                "repair: the fixer starts from the failed verification on {}",
5796                short(&head)
5797            ),
5798        );
5799        let (_job, seat, out) = self
5800            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5801                prompt: fix_prompt.clone(),
5802                spec,
5803                seat,
5804                cwd: wt.clone(),
5805                timeout,
5806                allow_write: true,
5807                sessions,
5808                artifacts: artifacts.clone(),
5809                stem: "fix-recovery".to_owned(),
5810                handover: None,
5811            })
5812            .await;
5813        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5814            rec.repair = Repair::WaitingForRestore { before: head };
5815            self.state.recovery = Some(rec);
5816            return self.stop_for_resource();
5817        }
5818        let failed = match &out {
5819            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5820            AgentOutcome::Quota(o) => {
5821                self.state.quota.push(QuotaLoss {
5822                    seat: seat.key.clone(),
5823                    node: "fix".to_owned(),
5824                    at: Timestamp::now(),
5825                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5826                });
5827                Some("rate limited (quota); the fixer could not run".to_owned())
5828            }
5829            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5830            AgentOutcome::Failed(e) => Some(e.clone()),
5831        };
5832        self.state.seats.insert(seat.key.clone(), seat);
5833        let coordination = self.coordinate(&wt).await?;
5834        let failed = match failed {
5835            // The agent's partial edits are not kept.
5836            Some(why) => {
5837                if let Err(e) = git::restore_to(&wt, &head).await {
5838                    return self
5839                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5840                        .await;
5841                }
5842                Some(why)
5843            }
5844            None => {
5845                if let Ok(r) = git::rescue_commit(
5846                    &wt,
5847                    "magi: recovery repair of the failed verification (uncommitted work)",
5848                )
5849                .await
5850                {
5851                    self.state.note_withheld("fix", &r.withheld);
5852                }
5853                None
5854            }
5855        };
5856        let after = git::rev_parse(&wt, "HEAD").await?;
5857        drop(coordination);
5858        self.state.event(
5859            "recovery",
5860            match &failed {
5861                Some(why) => format!("repair: nothing kept ({why})"),
5862                None if after != head => {
5863                    format!("repair: committed {}", short(&after))
5864                }
5865                None => "repair: the fixer made no commit".to_owned(),
5866            },
5867        );
5868        rec.repair = Repair::Done {
5869            before: head.clone(),
5870            committed: after != head,
5871            after,
5872            failed,
5873        };
5874        self.state.recovery = Some(rec);
5875        self.state.save()?;
5876        Ok(())
5877    }
5878
5879    /// A repair whose rollback could not be confirmed: the run is blocked and
5880    /// resumable, and the unconfirmed state is never reviewed.
5881    async fn stop_recovery(
5882        &mut self,
5883        mut rec: crate::recovery::RecoveryRecord,
5884        head: &str,
5885        why: String,
5886    ) -> Result<()> {
5887        // `Started` stays recorded: a resume retries the restore.
5888        rec.repair = crate::recovery::Repair::Started {
5889            before: head.to_owned(),
5890        };
5891        self.state.recovery = Some(rec);
5892        self.state
5893            .event("recovery", format!("repair could not be undone: {why}"));
5894        self.state.status = RunStatus::Blocked;
5895        self.state.save()?;
5896        bail!("recovery repair could not be rolled back: {why}")
5897    }
5898
5899    async fn review_loop(&mut self) -> Result<()> {
5900        loop {
5901            if self
5902                .state
5903                .base_sync
5904                .as_ref()
5905                .is_some_and(|s| s.conflict.is_some())
5906            {
5907                return Ok(());
5908            }
5909            if self.catchup_repairable() {
5910                let record = self.state.reviews.last().expect("repairable round").clone();
5911                let winner = self
5912                    .state
5913                    .winner()
5914                    .cloned()
5915                    .context("catch-up repair has no winner")?;
5916                if !self.fix_review_round(&winner, record).await? {
5917                    return Ok(());
5918                }
5919            }
5920            self.review_loop_pass().await?;
5921            if !self.catchup_repairable() {
5922                return Ok(());
5923            }
5924        }
5925    }
5926
5927    /// Derive an owed catch-up repair from durable round evidence, including
5928    /// reentry after a crash before the fixer was dispatched.
5929    fn catchup_repairable(&self) -> bool {
5930        self.state.reviews.len() < self.state.config.graph.review_rounds
5931            && self.state.reviews.last().is_some_and(|r| {
5932                !r.clean
5933                    && r.blocking == 0
5934                    && r.fix.is_none()
5935                    && r.e2e_status() == E2eStatus::Failed
5936            })
5937    }
5938
5939    async fn review_loop_pass(&mut self) -> Result<()> {
5940        // A base that would not rebase is a person's decision, not a review
5941        // round: nothing here would change the answer, and reviewers and a
5942        // fixer would be spending real budget on a tree that cannot land
5943        // regardless of what they find.
5944        if self
5945            .state
5946            .base_sync
5947            .as_ref()
5948            .is_some_and(|s| s.conflict.is_some())
5949        {
5950            return Ok(());
5951        }
5952        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5953        // agent files with `magi task add` name the run that paid for it. The
5954        // prompt overlay is cloned alongside it because the waves borrow it
5955        // while `self` is mutably borrowed by the node's own bookkeeping.
5956        let run_id = self.state.id.clone();
5957        let prompts = self.state.config.prompts.clone();
5958        let Some(winner) = self.state.winner().cloned() else {
5959            return Ok(());
5960        };
5961        let max_rounds = self.state.config.graph.review_rounds;
5962        // A clean round, an exhausted round budget, or a stalled tree (see
5963        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5964        // they are recorded — recomputed here, not read off `status`, so a
5965        // reentry into a run that already stopped restates the identical
5966        // verdict instead of silently handing back whatever an earlier node
5967        // in this same walk clobbered `status` to (a solo-candidate
5968        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5969        // below runs an empty range once the budget is spent, and would
5970        // otherwise fall through without touching `status` at all.
5971        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5972            // A reentry after a crash between the last round's save and
5973            // `stop_reviewing` reaches the hand-off here, not there.
5974            if status == RunStatus::Gating {
5975                self.record_contested_handoff();
5976            }
5977            self.state.status = status;
5978            self.state.save()?;
5979            return Ok(());
5980        }
5981        self.state.status = RunStatus::Reviewing;
5982        // A last recorded round whose own verification never resolved
5983        // (`ResourceBlocked` — the shared build cache, not the patch) is
5984        // never a concluded round, whatever the round budget says: starting
5985        // a fresh round on top of it would spend a whole new reviewer wave
5986        // re-reading an unchanged patch instead of just retrying the one
5987        // check that actually needs it, and once the budget is spent the
5988        // loop below has nothing left to do at all (its range is empty).
5989        // Retry that check directly instead, exactly the same retry
5990        // `stop_reviewing` already does for its own catch-up case.
5991        if self.state.reviews.last().is_some_and(|r| {
5992            r.e2e_status() == E2eStatus::ResourceBlocked
5993                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5994        }) {
5995            let shell = self.state.config.shell();
5996            return self
5997                .stop_unresolved_review(
5998                    "the last round's own verification never resolved",
5999                    &shell,
6000                    &winner.worktree,
6001                )
6002                .await;
6003        }
6004
6005        self.recovery_repair(&winner).await?;
6006        if self.resource_halt()? {
6007            return Ok(());
6008        }
6009
6010        let repo = self.state.repo.clone();
6011        let root = self.state.worktree_root();
6012        let language = self.state.config.graph.language.clone();
6013        let sessions = self.state.config.graph.sessions;
6014        let artifacts = self.merge_repair_artifacts();
6015        let base = self.landing_base();
6016        let base_short = short(&base);
6017        let reviewers = self.roles.reviewers.clone();
6018        let shell = self.state.config.shell();
6019
6020        for round in (self.state.reviews.len() + 1)..=max_rounds {
6021            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6022            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6023            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6024            // The prior round's own record, already persisted — never a
6025            // hand-carried variable of just its failing output: that is
6026            // exactly what let a round's e2e result drift out of sync with
6027            // which commit it was actually about (see `SCHEMA`'s doc for
6028            // schema 8). Judged against `head`, the commit reviewers are
6029            // about to look at now, so the summary always reads as "an
6030            // earlier head" here — this round's own patch has not been
6031            // checked yet.
6032            let prev_verification = self
6033                .state
6034                .reviews
6035                .last()
6036                .and_then(|r| r.verification_summary(&head));
6037            // Only the first round of a run that took a branch over: later
6038            // rounds have this run's own history.
6039            let prior_section = match (&self.state.recovery, round) {
6040                (Some(rec), 1) if self.state.reviews.is_empty() => {
6041                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6042                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6043                            Some(crate::recovery::head_of(
6044                                &git::diff(&winner.worktree, reviewed, "HEAD")
6045                                    .await
6046                                    .unwrap_or_default(),
6047                                crate::recovery::SINCE_DIFF_MAX,
6048                            ))
6049                        }
6050                        _ => None,
6051                    };
6052                    Some(prompt::recovery_review(rec, since.as_deref()))
6053                }
6054                _ => None,
6055            };
6056
6057            // Each reviewer gets its own detached checkout of exactly this
6058            // commit: nobody can perturb the winner's tree, and the fixer can
6059            // keep working without racing a reviewer.
6060            let mut jobs = Vec::new();
6061            let coordination = self.coordinate(&repo).await?;
6062            for (r, spec) in reviewers.iter().cloned().enumerate() {
6063                let wt = root.join(format!("review-{}", r + 1));
6064                // `clean -fdx` in a reset wipes setup's products, so every
6065                // round sets the seat up again.
6066                if wt.exists() {
6067                    git::reset_detached(&wt, &head).await?;
6068                } else {
6069                    git::worktree_add_detached(&repo, &wt, &head).await?;
6070                }
6071                self.setup_seat_worktree(&repo, &wt, false).await?;
6072                let seat_key = format!("review-{}", r + 1);
6073                // The seat starts the round on whoever answered it last, not
6074                // on the agent the spec names, so a failure is not re-paid.
6075                let spec = pick_start_spec(
6076                    &self.roles.reviewer_roster,
6077                    spec,
6078                    self.state.seat_history.get(&seat_key),
6079                );
6080                let seat = self.seat(&seat_key, &spec.id);
6081                jobs.push(SeatJob {
6082                    prompt: prompt::review(&prompt::ReviewCtx {
6083                        instruction: &self.state.instruction,
6084                        branch: &winner.branch,
6085                        base_short: &base_short,
6086                        stat: &stat,
6087                        patch: &patch,
6088                        verification: prev_verification.as_ref(),
6089                        prior: prior_section.as_deref(),
6090                        reviewers: reviewers.len(),
6091                        round,
6092                        rounds: max_rounds,
6093                        // A review-only run has no rankings, so nothing
6094                        // competed for this patch and the reviewer is told so.
6095                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6096                        lens: Lens::for_seat(r),
6097                        language: &language,
6098                        github_language: &self.state.config.graph.github_language,
6099                    }),
6100                    spec,
6101                    seat,
6102                    cwd: wt,
6103                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6104                    allow_write: false,
6105                    sessions,
6106                    artifacts: artifacts.clone(),
6107                    stem: format!("review-{round}-{}", r + 1),
6108                    handover: None,
6109                });
6110            }
6111
6112            drop(coordination);
6113            self.state.event(
6114                "review",
6115                format!(
6116                    "round {round}: {} reviewers on {}",
6117                    jobs.len(),
6118                    short(&head)
6119                ),
6120            );
6121            let mut quota_losses = Vec::new();
6122            let review_retries = self.state.config.graph.retries;
6123            let review_cache = self.state.config.cache_dir();
6124            let ctx = WaveCtx {
6125                carry_seats: true,
6126                pauses: Some((&self.pause, &self.interrupt)),
6127                run: &run_id,
6128                node: "review",
6129                prompts: &prompts,
6130                cache: review_cache.as_deref(),
6131                round: Some(round),
6132            };
6133            let results = ask_json_wave::<Review>(
6134                jobs,
6135                Arc::clone(&self.sem),
6136                review_retries,
6137                &self.roles.reviewer_roster,
6138                &ctx,
6139                &mut quota_losses,
6140                &mut self.state,
6141                &|_: &Review| Ok(()),
6142            )
6143            .await;
6144            if self.state.resource_block.is_some() {
6145                return self.stop_for_resource();
6146            }
6147            // Counted before the move below: how many of *this* round's
6148            // reviewer seats were lost to their own rate limit, as opposed to
6149            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6150            let round_quota_missing = quota_losses.len();
6151            self.state.quota.extend(quota_losses);
6152
6153            let mut records = Vec::new();
6154            let mut all_findings = Vec::new();
6155            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6156                let agent_id = seat.agent.clone();
6157                self.state.seats.insert(seat.key.clone(), seat);
6158                let mut record = ReviewRecord {
6159                    reviewer: r + 1,
6160                    agent: agent_id,
6161                    summary: String::new(),
6162                    findings: Vec::new(),
6163                    vote: None,
6164                    failed: None,
6165                    duration_ms: 0,
6166                    // Set for both outcomes: `failed: Some(_)` with
6167                    // `attempts > 0` is a seat every retry still lost, not a
6168                    // recovered one — only `failed: None` with `attempts > 0`
6169                    // reads as "answered after a nudge" (see this field's own
6170                    // doc).
6171                    attempts,
6172                };
6173                match res {
6174                    Ok((review, out)) => {
6175                        // Sanitized here, at the point every other piece of
6176                        // agent prose in this file is (candidate summaries,
6177                        // deliberation turns, vote reasons): a reviewer's own
6178                        // words are the one thing about it that could name
6179                        // it, and reconsideration below broadcasts this same
6180                        // summary and these same findings to every other
6181                        // seat on the panel.
6182                        record.summary =
6183                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6184                        record.vote = Some(review.vote);
6185                        record.duration_ms = out.duration_ms;
6186                        for (n, mut f) in review.findings.into_iter().enumerate() {
6187                            // ids are magi's, never the agent's: the fixer's
6188                            // adoption report is keyed by them.
6189                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6190                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6191                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6192                            // `file` is agent-supplied prose too, never
6193                            // checked against the real tree — the same
6194                            // exposure `title`/`detail` above have, just in
6195                            // a field easy to forget because it looks like a
6196                            // path rather than free text.
6197                            f.file = f
6198                                .file
6199                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6200                            all_findings.push(f.clone());
6201                            record.findings.push(f);
6202                        }
6203                        self.state.event(
6204                            "review",
6205                            format!(
6206                                "round {round}: reviewer {} voted {} with {} finding(s)",
6207                                r + 1,
6208                                review.vote.label(),
6209                                record.findings.len()
6210                            ),
6211                        );
6212                    }
6213                    Err(e) => {
6214                        record.failed = Some(e.to_string());
6215                        self.state.event(
6216                            "review",
6217                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6218                        );
6219                    }
6220                }
6221                records.push(record);
6222            }
6223
6224            // Tally the round's votes and, if they split, spend the one
6225            // round of reconsideration the split -> deliberate -> revote
6226            // shape `judge`/`vote` use for the panel, sized down to what a
6227            // read-only review round can afford: one round, and a revote
6228            // rather than an argument, because the panel already wrote its
6229            // reasoning down as findings the first time around.
6230            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6231            let vote_split =
6232                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6233            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6234            if vote_split {
6235                self.state.event(
6236                    "review",
6237                    format!(
6238                        "round {round}: votes split ({}) — one round of reconsideration",
6239                        initial_votes
6240                            .iter()
6241                            .map(|v| v.label())
6242                            .collect::<Vec<_>>()
6243                            .join(", ")
6244                    ),
6245                );
6246                // Seats read every seat's findings and votes, still numbered
6247                // and never named — the same anonymity `review` itself keeps.
6248                let panel: Vec<ReviewSeatReport<'_>> = records
6249                    .iter()
6250                    .filter_map(|r| {
6251                        r.vote.map(|vote| ReviewSeatReport {
6252                            reviewer: r.reviewer,
6253                            vote,
6254                            summary: &r.summary,
6255                            findings: &r.findings,
6256                        })
6257                    })
6258                    .collect();
6259
6260                let mut jobs = Vec::new();
6261                let mut seats_at = Vec::new();
6262                for (r, spec) in reviewers.iter().cloned().enumerate() {
6263                    // A seat with no initial vote has nothing to reconsider
6264                    // from and stays absent, the same as it stayed absent
6265                    // from `panel` above.
6266                    if records[r].vote.is_none() {
6267                        continue;
6268                    }
6269                    let wt = root.join(format!("review-{}", r + 1));
6270                    let seat_key = format!("review-{}", r + 1);
6271                    let spec = self.occupant(&seat_key, spec);
6272                    let seat = self.seat(&seat_key, &spec.id);
6273                    // A seat with no live session has already forgotten the
6274                    // initial review's prompt — restate the patch it is
6275                    // voting on, the same as `deliberate`/`vote` do for a
6276                    // judge in the same position.
6277                    // The panel already carries this seat's own review and
6278                    // vote, so restating the patch makes the prompt whole for
6279                    // a seat handed to another agent.
6280                    let build = |with_patch: bool| {
6281                        prompt::review_reconsider(&ReviewReconsiderCtx {
6282                            instruction: &self.state.instruction,
6283                            reviewer: r + 1,
6284                            lens: Lens::for_seat(r),
6285                            panel: &panel,
6286                            patch: with_patch.then_some(ReviewPatch {
6287                                branch: &winner.branch,
6288                                base_short: &base_short,
6289                                stat: &stat,
6290                                patch: &patch,
6291                            }),
6292                            round,
6293                            rounds: max_rounds,
6294                            language: &language,
6295                        })
6296                    };
6297                    let full = build(true);
6298                    let prompt = if has_context(&spec, &seat, sessions) {
6299                        build(false)
6300                    } else {
6301                        full.clone()
6302                    };
6303                    jobs.push(SeatJob {
6304                        prompt,
6305                        spec,
6306                        seat,
6307                        cwd: wt,
6308                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6309                        allow_write: false,
6310                        sessions,
6311                        artifacts: artifacts.clone(),
6312                        stem: format!("review-{round}-reconsider-{}", r + 1),
6313                        handover: Some(full),
6314                    });
6315                    seats_at.push(r);
6316                }
6317
6318                let mut recon_quota_losses = Vec::new();
6319                let recon_cache = self.state.config.cache_dir();
6320                let recon_ctx = WaveCtx {
6321                    carry_seats: true,
6322                    pauses: Some((&self.pause, &self.interrupt)),
6323                    run: &run_id,
6324                    node: "review",
6325                    prompts: &prompts,
6326                    cache: recon_cache.as_deref(),
6327                    round: Some(round),
6328                };
6329                let recon_results = ask_json_wave::<ReviewRevote>(
6330                    jobs,
6331                    Arc::clone(&self.sem),
6332                    review_retries,
6333                    &self.roles.reviewer_roster,
6334                    &recon_ctx,
6335                    &mut recon_quota_losses,
6336                    &mut self.state,
6337                    &|_: &ReviewRevote| Ok(()),
6338                )
6339                .await;
6340                if self.state.resource_block.is_some() {
6341                    return self.stop_for_resource();
6342                }
6343                self.state.quota.extend(recon_quota_losses);
6344
6345                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6346                    let agent_id = seat.agent.clone();
6347                    self.state.seats.insert(seat.key.clone(), seat);
6348                    let mut rec = ReviewRevoteRecord {
6349                        reviewer: r + 1,
6350                        agent: agent_id,
6351                        vote: None,
6352                        reason: String::new(),
6353                        failed: None,
6354                    };
6355                    match res {
6356                        Ok((rv, _)) => {
6357                            rec.vote = Some(rv.vote);
6358                            rec.reason =
6359                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6360                            self.state.event(
6361                                "review",
6362                                format!(
6363                                    "round {round}: reviewer {} revoted {}",
6364                                    r + 1,
6365                                    rv.vote.label()
6366                                ),
6367                            );
6368                        }
6369                        Err(e) => {
6370                            rec.failed = Some(e.to_string());
6371                            self.state.event(
6372                                "review",
6373                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6374                            );
6375                        }
6376                    }
6377                    reconsideration.push(rec);
6378                }
6379            } else if initial_votes.len() > 1 {
6380                self.state.event(
6381                    "review",
6382                    format!(
6383                        "round {round}: votes agreed ({}) — no reconsideration",
6384                        initial_votes[0].label()
6385                    ),
6386                );
6387            }
6388
6389            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6390            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6391            // A round that already has a blocking finding and a round left to
6392            // try is going back to the fixer no matter what `verify.e2e`
6393            // says, so running it first only spends the loop's slowest step
6394            // (minutes, for a Rust repo's full test suite) on a head about
6395            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6396            // runs once a round has no blocking findings left (see
6397            // `round_is_clean`, which a deferred — empty — `e2e` can never
6398            // satisfy since `blocking` is nonzero whenever this branch is
6399            // taken), and `stop_reviewing` forces a real run before it will
6400            // ever read a deferred round as green.
6401            let defer_e2e =
6402                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6403            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6404                let reason =
6405                    format!("{blocking} blocking finding(s) already required a fix this round");
6406                self.state.event(
6407                    "verify",
6408                    format!(
6409                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6410                         {}); it will run once a round has none left",
6411                        short(&head)
6412                    ),
6413                );
6414                (Vec::new(), false, true, Some(reason))
6415            } else {
6416                let e2e_commands = self.state.config.verify.e2e.clone();
6417                let cache_dir = self.state.config.cache_dir();
6418                let context = format!("round {round}");
6419                let wait = self.repository_wait();
6420                let (e2e, verify_retried) = with_cache_lease(
6421                    &mut self.state,
6422                    cache_dir.as_deref(),
6423                    "e2e",
6424                    "e2e",
6425                    &winner.worktree,
6426                    &head,
6427                    verify_timeout,
6428                    &context,
6429                    &wait,
6430                    |state, budget| {
6431                        let shell = shell.clone();
6432                        let e2e_commands = e2e_commands.clone();
6433                        let worktree = winner.worktree.clone();
6434                        let context = context.clone();
6435                        async move {
6436                            run_e2e_with_retry(
6437                                state,
6438                                &shell,
6439                                &e2e_commands,
6440                                &worktree,
6441                                budget,
6442                                &context,
6443                            )
6444                            .await
6445                        }
6446                    },
6447                )
6448                .await?;
6449                (e2e, verify_retried, false, None)
6450            };
6451
6452            let expected = records.len();
6453            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6454            let incomplete = answered < expected;
6455            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6456            let policy = self.state.config.graph.incomplete_review;
6457            let clean = round_is_clean(
6458                blocking,
6459                e2e_ok,
6460                answered,
6461                expected,
6462                round_quota_missing,
6463                policy,
6464            );
6465
6466            let mut round_record = ReviewRound {
6467                round,
6468                head: head.clone(),
6469                verified_head: None,
6470                verified_at: None,
6471                reviews: records,
6472                e2e,
6473                verify_retried,
6474                e2e_deferred,
6475                e2e_defer_reason,
6476                fix: None,
6477                blocking,
6478                answered,
6479                expected,
6480                clean,
6481                progressed: false,
6482                vote_split,
6483                reconsideration,
6484                verdict: None,
6485            };
6486            // The final vote per seat is its revote where reconsideration
6487            // ran and answered, its initial vote otherwise — the same
6488            // fallback `tally` uses for a judge whose private vote failed.
6489            round_record.verdict = ReviewVote::worst(
6490                round_record
6491                    .final_votes()
6492                    .into_iter()
6493                    .map(|(_, _, vote)| vote),
6494            );
6495            // Which commit and when magi actually attempted to check —
6496            // known the moment a command was dispatched against `head`,
6497            // whether or not it finished: a resource-blocked attempt still
6498            // targeted a specific commit at a specific time, and leaving
6499            // that unrecorded is exactly what made `verification_summary`
6500            // report a fresh attempt as "commit unknown ... recorded before
6501            // this was tracked", indistinguishable from a genuinely old,
6502            // untracked record. Only a deferred or unconfigured round never
6503            // ran at all and has nothing to record — see
6504            // `ReviewRound::verified_head`'s own doc.
6505            if !matches!(
6506                round_record.e2e_status(),
6507                E2eStatus::Deferred | E2eStatus::NotConfigured
6508            ) {
6509                round_record.verified_head = Some(head.clone());
6510                round_record.verified_at = Some(Timestamp::now());
6511            }
6512
6513            if incomplete {
6514                let missing: Vec<String> = round_record
6515                    .reviews
6516                    .iter()
6517                    .filter(|r| r.failed.is_some())
6518                    .map(|r| format!("review-{}", r.reviewer))
6519                    .collect();
6520                self.state.event(
6521                    "review",
6522                    format!(
6523                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6524                        missing.join(", ")
6525                    ),
6526                );
6527            }
6528
6529            if clean {
6530                self.state.event(
6531                    "review",
6532                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6533                        format!(
6534                            "round {round}: clean (warn policy, incomplete panel) — no \
6535                             blocking findings from the seats that answered, verification green"
6536                        )
6537                    } else if incomplete {
6538                        format!(
6539                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6540                             quorum) — no blocking findings from the seats that answered, \
6541                             verification green",
6542                            expected - answered
6543                        )
6544                    } else {
6545                        format!("round {round}: clean — no blocking findings, verification green")
6546                    },
6547                );
6548                self.state.reviews.push(round_record);
6549                self.state.status = RunStatus::Gating;
6550                self.state.save()?;
6551                return Ok(());
6552            }
6553
6554            // Nothing was raised and verification passed, but not every seat
6555            // answered and `round_is_clean` still refused to call it clean —
6556            // either a seat is missing for a reason other than its own quota
6557            // (a crash, a timeout, unparsable output — worth another try), or
6558            // every seat that could have answered lost its quota and nobody
6559            // is left to decide on: re-review rather than send the fixer
6560            // after a round with nothing to fix.
6561            if incomplete && blocking == 0 && e2e_ok {
6562                self.state.reviews.push(round_record);
6563                self.state.save()?;
6564                if round == max_rounds {
6565                    self.state.status = RunStatus::Blocked;
6566                    self.state.event(
6567                        "review",
6568                        format!(
6569                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6570                             refusing to call it clean",
6571                            expected - answered
6572                        ),
6573                    );
6574                    return Ok(());
6575                }
6576                continue;
6577            }
6578
6579            // Nothing for the fixer to act on (`blocking == 0`) and the only
6580            // reason this round is not clean is that magi itself never got
6581            // a command to run — the shared build cache, not the patch (see
6582            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6583            // the fixer would invite a change to appease contention that has
6584            // nothing to do with the diff, and would leave this attempt
6585            // sitting in the next round's prompt as if it were about an
6586            // earlier, superseded commit rather than what it actually is:
6587            // the same head, still waiting to be checked. Wait for it the
6588            // same way the final round's own contention is already handled,
6589            // whatever round this happens to be.
6590            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6591                self.state.reviews.push(round_record);
6592                return self
6593                    .stop_unresolved_review(
6594                        "the round's own verification could not run",
6595                        &shell,
6596                        &winner.worktree,
6597                    )
6598                    .await;
6599            }
6600
6601            if round == max_rounds {
6602                self.state.reviews.push(round_record);
6603                return self
6604                    .stop_reviewing(
6605                        &format!(
6606                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6607                        ),
6608                        &shell,
6609                        &winner.worktree,
6610                    )
6611                    .await;
6612            }
6613
6614            if !self.fix_review_round(&winner, round_record).await? {
6615                return Ok(());
6616            }
6617        }
6618        Ok(())
6619    }
6620
6621    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6622    /// False means admission or continuation policy stopped the review loop.
6623    async fn fix_review_round(
6624        &mut self,
6625        winner: &Candidate,
6626        mut round_record: ReviewRound,
6627    ) -> Result<bool> {
6628        let round = round_record.round;
6629        let shell = self.state.config.shell();
6630        let step = self.run_fix_round(winner, &mut round_record).await?;
6631        if matches!(step, FixStep::Resource) {
6632            self.stop_for_resource()?;
6633            return Ok(false);
6634        }
6635        if self.state.reviews.len() < round {
6636            self.state.reviews.push(round_record);
6637        } else {
6638            self.state.reviews[round - 1] = round_record;
6639        }
6640        self.state.save()?;
6641
6642        // The fixer's own report never came back this round, even after
6643        // `continue_fix_report`'s own budget was spent on it — not an
6644        // ordinary "no report" (dropped stream, quota, plain failure),
6645        // which already reads that way and is left to the existing round
6646        // budget. Stopping here, rather than opening another round, is
6647        // what keeps a next reviewer/fixer wave from ever being
6648        // dispatched onto `winner.worktree` while whatever the seat's
6649        // last call may still have running there is unaccounted for: no
6650        // process liveness check exists (and none is being added — see
6651        // AGENTS.md/this task's own scope), so the only way to honour
6652        // "nothing starts before a valid report returns" is to not start
6653        // anything further on this worktree from this run at all.
6654        if matches!(step, FixStep::Recorded { report_lost: true }) {
6655            self.stop_reviewing(
6656                "the fixer's adoption report never came back, even after resuming its \
6657                         own seat; refusing to start another round against the same worktree \
6658                         while that is unresolved",
6659                &shell,
6660                &winner.worktree,
6661            )
6662            .await?;
6663            return Ok(false);
6664        }
6665
6666        let streak = self
6667            .state
6668            .reviews
6669            .iter()
6670            .rev()
6671            .take_while(|r| !r.progressed)
6672            .count();
6673        if streak >= STAGNANT_LIMIT {
6674            self.stop_reviewing(
6675                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6676                &shell,
6677                &winner.worktree,
6678            )
6679            .await?;
6680            return Ok(false);
6681        }
6682        Ok(true)
6683    }
6684
6685    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6686    async fn run_fix_round(
6687        &mut self,
6688        winner: &Candidate,
6689        round_record: &mut ReviewRound,
6690    ) -> Result<FixStep> {
6691        let round = round_record.round;
6692        let max_rounds = self.state.config.graph.review_rounds;
6693        let run_id = self.state.id.clone();
6694        let prompts = self.state.config.prompts.clone();
6695        let language = self.state.config.graph.language.clone();
6696        let sessions = self.state.config.graph.sessions;
6697        let artifacts = self.merge_repair_artifacts();
6698        let base = self.landing_base();
6699        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6700        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6701        let this_round_verification = round_record.verification_summary(&head);
6702        let all_findings: Vec<_> = round_record
6703            .reviews
6704            .iter()
6705            .flat_map(|r| r.findings.iter().cloned())
6706            .collect();
6707        // Fix. The winner's own implementer seat continues its conversation:
6708        // the competition is over, so context is pure benefit now.
6709        let blocking_findings: Vec<_> = all_findings
6710            .iter()
6711            .filter(|f| f.severity.blocks())
6712            .cloned()
6713            .collect();
6714        let fix_prompt = prompt::fix_in(
6715            &self.state.instruction,
6716            &blocking_findings,
6717            this_round_verification.as_ref(),
6718            round,
6719            max_rounds,
6720            &language,
6721            &self.state.config.graph.github_language,
6722        );
6723        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6724        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6725        let (job, seat, out) = self
6726            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6727                prompt: fix_prompt.clone(),
6728                spec,
6729                seat,
6730                cwd: winner.worktree.clone(),
6731                timeout,
6732                allow_write: true,
6733                sessions,
6734                artifacts: artifacts.clone(),
6735                stem: format!("fix-{round}"),
6736                handover: None,
6737            })
6738            .await;
6739        // Not asked: the round is not recorded (so a resume repeats it),
6740        // no fixer was handed over to, and the seat is left as it was.
6741        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6742            return Ok(FixStep::Resource);
6743        }
6744        let agent_id = seat.agent.clone();
6745
6746        let mut fix = FixRecord {
6747            agent: agent_id,
6748            addressed: Vec::new(),
6749            rejected: Vec::new(),
6750            notes: String::new(),
6751            committed: false,
6752            failed: None,
6753            duration_ms: 0,
6754            continuation: None,
6755        };
6756        let mut continuation = ContinuationRecord::not_needed();
6757        let mut final_seat = seat.clone();
6758        match out {
6759            AgentOutcome::Ok(o) => {
6760                fix.duration_ms = o.duration_ms;
6761                let parsed = verdict::extract_json::<FixReport>(&o.text);
6762                // A parsed report standing next to a command this same
6763                // reply's own CLI never confirmed the exit status of is
6764                // not a resolved answer — the identical `CommandEvidence`
6765                // `state.jobs` renders, read here instead of only on
6766                // display, per the completion judgment and the shown
6767                // record needing to agree.
6768                let incomplete_reason = match &parsed {
6769                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6770                        "the reply parsed, but it reported a command whose own CLI never \
6771                             confirmed an exit status"
6772                            .to_owned(),
6773                    ),
6774                    Ok(_) => None,
6775                    Err(e) => Some(e.to_string()),
6776                };
6777                match incomplete_reason {
6778                    None => {
6779                        let report = parsed.expect("checked Ok above");
6780                        fix.addressed = report.addressed;
6781                        fix.rejected = report.rejected;
6782                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6783                    }
6784                    Some(reason) => {
6785                        let (resumed_seat, resolved, failure, cont) = self
6786                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6787                            .await;
6788                        fix.duration_ms += cont.cumulative_wait_ms;
6789                        continuation = cont;
6790                        final_seat = resumed_seat;
6791                        match resolved {
6792                            Some(report) => {
6793                                fix.addressed = report.addressed;
6794                                fix.rejected = report.rejected;
6795                                fix.notes =
6796                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6797                            }
6798                            None => fix.failed = failure,
6799                        }
6800                    }
6801                }
6802            }
6803            // The CLI's raw error JSON is not a fix report to parse.
6804            AgentOutcome::Dropped(o) => {
6805                fix.duration_ms = o.duration_ms;
6806                let why = o
6807                    .dropped
6808                    .as_ref()
6809                    .map(|d| d.why.as_str())
6810                    .unwrap_or("the CLI ended the stream without delivering its answer");
6811                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6812            }
6813            AgentOutcome::Quota(o) => {
6814                self.state.quota.push(QuotaLoss {
6815                    seat: final_seat.key.clone(),
6816                    node: "fix".to_owned(),
6817                    at: Timestamp::now(),
6818                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6819                });
6820                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6821            }
6822            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6823        }
6824        fix.continuation = Some(continuation);
6825        self.state.seats.insert(final_seat.key.clone(), final_seat);
6826        let coordination = self.coordinate(&winner.worktree).await?;
6827        if let Ok(r) = git::rescue_commit(
6828            &winner.worktree,
6829            &format!("magi: review round {round} fixes (uncommitted work)"),
6830        )
6831        .await
6832        {
6833            self.state.note_withheld("fix", &r.withheld);
6834        }
6835        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6836        fix.committed = after != before;
6837        // Judged by what `git` says moved against base, never by the
6838        // fixer's own `addressed`/`rejected` count — see
6839        // `ReviewRound::progressed`. Propagated with `?`, the same as the
6840        // `patch` snapshot above: swallowing this error would default
6841        // `diff_after` to empty, which almost always differs from a
6842        // non-empty `patch` and reads as "progressed" — exactly backwards
6843        // for a `git` failure the stagnation check cannot see through.
6844        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6845        drop(coordination);
6846        let progressed = diff_after != patch;
6847        let commit_note = if fix.committed {
6848            "committed"
6849        } else {
6850            "NO new commit"
6851        };
6852        let tree_note = if progressed {
6853            "changed vs base"
6854        } else {
6855            "unchanged vs base"
6856        };
6857        self.state.event(
6858            "fix",
6859            match &fix.failed {
6860                // Distinct on purpose from "0 addressed, 0 rejected": the
6861                // fixer's own diff still landed (blocking counts do keep
6862                // falling round over round), only its adoption report did
6863                // not come back, so this must never read like every
6864                // finding was reviewed and declined.
6865                Some(reason) => {
6866                    format!(
6867                        "round {round}: fixer's adoption report was lost ({reason}); \
6868                             {commit_note}, tree {tree_note}"
6869                    )
6870                }
6871                None => format!(
6872                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6873                         {tree_note}{}",
6874                    fix.addressed.len(),
6875                    fix.rejected.len(),
6876                    if continuation.outcome == ContinuationOutcome::Resumed {
6877                        format!(
6878                            " (adoption report recovered after {} continuation(s))",
6879                            continuation.attempts
6880                        )
6881                    } else {
6882                        String::new()
6883                    },
6884                ),
6885            },
6886        );
6887        round_record.fix = Some(fix);
6888        round_record.progressed = progressed;
6889        Ok(FixStep::Recorded {
6890            report_lost: matches!(
6891                continuation.outcome,
6892                ContinuationOutcome::Exhausted
6893                    | ContinuationOutcome::QuotaLost
6894                    | ContinuationOutcome::NoSession
6895            ),
6896        })
6897    }
6898
6899    /// Decide, from the last recorded round's own verification, whether
6900    /// stopping the review loop is a hand-off or a genuine block.
6901    ///
6902    /// Called once the loop has given up trying — the round budget is spent,
6903    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6904    /// findings still open, never while a round is still clean or the
6905    /// incomplete-panel case handled inline above. Gate and e2e are facts
6906    /// about the tree; a lingering review finding is an opinion, and this
6907    /// workload's own `magi stats` puts reviewer precision low enough
6908    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6909    /// must not by itself stand between a green, verified change and the
6910    /// human who decides what to do with it. A red e2e is not an opinion, so
6911    /// that case still blocks, with the failing command and a tail of its
6912    /// output recorded here rather than left in `run.json` for someone to go
6913    /// find.
6914    ///
6915    /// A round that deferred its own e2e (see [`Config::graph`]'s
6916    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6917    /// only because nothing ran, and treating an empty list as a passing one
6918    /// here is exactly the "deferred painted green" bug this function exists
6919    /// to not have. When the last round's own verification never resolved —
6920    /// deferred on purpose, or a real attempt the shared build cache blocked
6921    /// — this makes (or retries) the real run, on the actual worktree this
6922    /// loop is about to stop touching, before deciding anything. A
6923    /// resource-blocked attempt is likewise never read as either green or
6924    /// red: it is evidence about the machine, not the patch (see
6925    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6926    /// blocked cache leaves this call without deciding rather than guessing
6927    /// — the caller retries on a later reentry.
6928    /// Record, once, that the review loop handed off over a blocking finding
6929    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6930    /// `land` asks the owner even with `land_approval` off. Called from every
6931    /// path that concludes `Gating`; a reentry keeps the first record.
6932    fn record_contested_handoff(&mut self) {
6933        if self.state.contested_handoff.is_some() {
6934            return;
6935        }
6936        let Some(contested) = self
6937            .state
6938            .reviews
6939            .last()
6940            .and_then(ReviewRound::contested_handoff)
6941        else {
6942            return;
6943        };
6944        self.state.event(
6945            "review",
6946            format!(
6947                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6948                 wait for the owner's approval",
6949                contested.findings.len(),
6950                contested.rejecters.len()
6951            ),
6952        );
6953        self.state.contested_handoff = Some(contested);
6954    }
6955
6956    async fn stop_unresolved_review(
6957        &mut self,
6958        why: &str,
6959        shell: &[String],
6960        worktree: &Path,
6961    ) -> Result<()> {
6962        self.stop_reviewing_inner(why, shell, worktree, true).await
6963    }
6964
6965    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6966        self.stop_reviewing_inner(why, shell, worktree, false).await
6967    }
6968
6969    async fn stop_reviewing_inner(
6970        &mut self,
6971        why: &str,
6972        shell: &[String],
6973        worktree: &Path,
6974        repairable: bool,
6975    ) -> Result<()> {
6976        let round_idx = self.state.reviews.len() - 1;
6977        // A deferred round and a resource-blocked one are the same shape
6978        // here: neither has a real result yet, and both get one more
6979        // attempt. Read off `e2e_status` — the single source for this —
6980        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6981        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6982        // still retries instead of being read as a settled result the
6983        // instant it stops being empty.
6984        let needs_catchup_run = matches!(
6985            self.state.reviews[round_idx].e2e_status(),
6986            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6987        );
6988        if needs_catchup_run {
6989            let round = self.state.reviews[round_idx].round;
6990            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6991            let commands = self.state.config.verify.e2e.clone();
6992            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6993            let cache_dir = self.state.config.cache_dir();
6994            let context = format!(
6995                "round {round}: verification unresolved, catching up before the final decision"
6996            );
6997            let wait = self.repository_wait();
6998            let (outcomes, verify_retried) = with_cache_lease(
6999                &mut self.state,
7000                cache_dir.as_deref(),
7001                "e2e",
7002                "e2e",
7003                worktree,
7004                &attempted_head,
7005                timeout,
7006                &context,
7007                &wait,
7008                |state, budget| {
7009                    let shell = shell.to_vec();
7010                    let commands = commands.clone();
7011                    let context = context.clone();
7012                    async move {
7013                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7014                            .await
7015                    }
7016                },
7017            )
7018            .await?;
7019            let last = &mut self.state.reviews[round_idx];
7020            last.e2e = outcomes;
7021            last.verify_retried = verify_retried;
7022            // Always the commit and time this attempt actually targeted,
7023            // whether or not it happens to equal the reviewed `head` and
7024            // whether or not a command finished — see
7025            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7026            // attempt is recorded too, so a later reader sees "attempted
7027            // again at T2" rather than silence.
7028            last.verified_head = Some(attempted_head);
7029            last.verified_at = Some(Timestamp::now());
7030            if verify_inconclusive(&last.e2e) {
7031                // Still not a real result: `e2e_deferred` is left exactly
7032                // as it was, so `needs_catchup_run` above reads
7033                // `ResourceBlocked` (via `e2e_status`, which checks
7034                // `resource_blocked` before `e2e_deferred`) and retries
7035                // again on the next reentry, rather than recording
7036                // contention as a red e2e and blocking the run on it.
7037                self.state.save()?;
7038                return Ok(());
7039            }
7040            last.e2e_deferred = false;
7041        }
7042        let last = &self.state.reviews[round_idx];
7043        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7044
7045        match last.e2e_status() {
7046            E2eStatus::Failed => {
7047                let red: Vec<String> = last
7048                    .e2e
7049                    .iter()
7050                    .filter(|o| !o.ok())
7051                    .map(|o| {
7052                        format!(
7053                            "`{}` -> {:?}\n{}",
7054                            o.command,
7055                            o.code,
7056                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7057                        )
7058                    })
7059                    .collect();
7060                let can_repair = last.blocking == 0 && last.fix.is_none();
7061                self.state.reviews[round_idx].clean = false;
7062                self.state
7063                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7064                self.state.status = RunStatus::Blocked;
7065                if repairable
7066                    && can_repair
7067                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7068                {
7069                    self.state.status = RunStatus::Reviewing;
7070                }
7071            }
7072            // `needs_catchup_run` above already retried once this call; if
7073            // it is still blocked, this is magi's own admission it could
7074            // not get a command to run, never a verdict on the patch — the
7075            // run is left exactly where a later reentry can retry again.
7076            E2eStatus::ResourceBlocked => {
7077                self.state.event(
7078                    "review",
7079                    format!(
7080                        "{why}; e2e could not run (resource unavailable); not \
7081                         deciding yet"
7082                    ),
7083                );
7084            }
7085            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7086                self.state.event(
7087                    "review",
7088                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7089                );
7090                self.record_contested_handoff();
7091                self.state.status = RunStatus::Gating;
7092            }
7093        }
7094        self.state.save()?;
7095        Ok(())
7096    }
7097
7098    // ----------------------------------------------------------------- gate
7099
7100    async fn gate(&mut self) -> Result<()> {
7101        // Judged by the review record itself, not by `status`: a solo
7102        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7103        // reentry (see `judge`), and trusting it here is exactly how a run
7104        // that exhausted its review budget got gated and merged a second
7105        // time around. `review_conclusion` recomputes the review loop's own
7106        // verdict from the round records themselves — `Gating` for a clean
7107        // round or a hand-off (see `stop_reviewing`), anything else means the
7108        // loop is still going or genuinely blocked.
7109        // A base the winner could not be replayed onto is a decision, not a
7110        // round: there is no landing tree to gate. Read as its own record for
7111        // the same reason the review verdict is.
7112        if self.state.status == RunStatus::Failed
7113            || self
7114                .state
7115                .base_sync
7116                .as_ref()
7117                .is_some_and(|s| s.conflict.is_some())
7118            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7119                != Some(RunStatus::Gating)
7120        {
7121            return Ok(());
7122        }
7123        if self.state.gate_ran {
7124            // `review_loop` derives its conclusion from the clean review
7125            // record on every reentry and therefore puts a completed run back
7126            // in `Gating`. A recorded gate is a stronger, terminal fact:
7127            // retain its original command output (or lack of any, for a repo
7128            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7129            // doc) and restore `Blocked` on a real failure rather than
7130            // pretending the command is still running or running it a second
7131            // time. `gate_ran == false` remains the only shape — unattempted,
7132            // or a resource-blocked retry — that may still need to execute a
7133            // command.
7134            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7135                self.state.status = RunStatus::Blocked;
7136                self.state.save()?;
7137            }
7138            return Ok(());
7139        }
7140        let Some(winner) = self.state.winner().cloned() else {
7141            return Ok(());
7142        };
7143        self.state.status = RunStatus::Gating;
7144        let mut outcomes = self.run_gate(&winner).await?;
7145        loop {
7146            // A resource-blocked outcome means the gate command never actually
7147            // ran - the shared build cache could not be acquired or confirmed
7148            // fresh in time - which is evidence about the machine, not about
7149            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7150            // Recording it as a red gate would mark a run `Blocked` on nothing
7151            // but contention magi has already logged; leaving `self.state.gate`
7152            // empty and `self.state.gate_ran` false instead keeps the shape
7153            // this function already treats as "still needs to run" (see the
7154            // early-return above), so the next call retries the command
7155            // rather than concluding anything.
7156            if verify_inconclusive(&outcomes) {
7157                self.state.save()?;
7158                return Ok(());
7159            }
7160            if outcomes.iter().all(CommandOutcome::ok) {
7161                break;
7162            }
7163            match self.gate_fix_round(&winner, &outcomes).await? {
7164                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7165                GateFix::Stop => break,
7166                GateFix::Defer => {
7167                    self.state.save()?;
7168                    return Ok(());
7169                }
7170            }
7171        }
7172        let passed = outcomes.iter().all(CommandOutcome::ok);
7173        self.state.gate = outcomes;
7174        self.state.gate_ran = true;
7175        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7176        if !passed {
7177            self.state.status = RunStatus::Blocked;
7178            let spent = self.state.gate_fixes.len();
7179            self.state.event(
7180                "gate",
7181                if spent == 0 {
7182                    "gate failed; not merging".to_owned()
7183                } else {
7184                    format!("gate failed after {spent} gate-fix round(s); not merging")
7185                },
7186            );
7187        }
7188        self.state.save()?;
7189        Ok(())
7190    }
7191
7192    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7193    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7194    /// after review is clean and never on a candidate awaiting judging.
7195    ///
7196    /// Never fails the run: a non-zero exit or timeout is a warning and a
7197    /// recorded outcome, and the gate remains the single arbiter. Nothing
7198    /// configured means nothing happens - no event, no commit. `commit_all`
7199    /// commits any leftover change under the neutral identity and returns
7200    /// `false` when the tree is clean, so no empty commit is ever made.
7201    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7202        let commands = state.config.verify.pre_gate.clone();
7203        if commands.is_empty() {
7204            return;
7205        }
7206        let shell = state.config.shell();
7207        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7208        let (outcomes, _) = run_commands(
7209            state,
7210            "pre_gate",
7211            "pre_gate",
7212            0,
7213            &shell,
7214            &commands,
7215            &winner.worktree,
7216            timeout,
7217        )
7218        .await;
7219        for o in &outcomes {
7220            if !o.ok() {
7221                tracing::warn!(
7222                    "pre_gate `{}` failed ({:?}); the gate decides",
7223                    o.command,
7224                    o.code
7225                );
7226            }
7227            state.event(
7228                "pre_gate",
7229                format!(
7230                    "`{}` -> {}",
7231                    o.command,
7232                    if o.ok() {
7233                        "pass".to_owned()
7234                    } else {
7235                        format!(
7236                            "FAIL ({:?})\n{}",
7237                            o.code,
7238                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7239                        )
7240                    }
7241                ),
7242            );
7243        }
7244        state.pre_gate = outcomes;
7245        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7246            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7247                Ok(head) => {
7248                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7249                    state.pre_gate_commit = Some(head);
7250                }
7251                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7252            },
7253            Ok(false) => {}
7254            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7255        }
7256        if let Err(e) = state.save() {
7257            tracing::warn!("could not persist the pre_gate record: {e:#}");
7258        }
7259    }
7260
7261    /// Run `verify.gate` once against the winner's current tree, logging one
7262    /// event per command. Empty when nothing is configured.
7263    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7264        let wait = self.repository_wait();
7265        run_landing_gate(&mut self.state, winner, &wait).await
7266    }
7267
7268    /// One bounded fix round for a failing gate.
7269    ///
7270    /// The fixer is told the failure came from the gate itself, not from a
7271    /// reviewer, and is shown the failed commands, their exit codes and a tail
7272    /// of their output - whatever `[verify].gate` holds, nothing here knows
7273    /// what those commands run. Only a normal non-zero exit that printed
7274    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7275    /// command or a full disk says nothing about the code, and a fixer sent
7276    /// after it can only appease the machine. The round is judged by what git
7277    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7278    /// again before the gate does, so a fix cannot trade a green gate for a
7279    /// red e2e unnoticed.
7280    async fn gate_fix_round(
7281        &mut self,
7282        winner: &Candidate,
7283        outcomes: &[CommandOutcome],
7284    ) -> Result<GateFix> {
7285        let cap = self.state.config.graph.gate_fix_rounds;
7286        let spent = self.state.gate_fixes.len();
7287        if spent >= cap {
7288            if cap > 0 {
7289                self.state.event(
7290                    "gate",
7291                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7292                );
7293            }
7294            return Ok(GateFix::Stop);
7295        }
7296        if !gate_fixable(outcomes) {
7297            self.state.event(
7298                "gate",
7299                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7300                 command or similar); not spending a fix round on it",
7301            );
7302            return Ok(GateFix::Stop);
7303        }
7304        let min_free = self.state.config.disk.min_free_bytes;
7305        if min_free > 0 {
7306            match crate::disk::free_bytes(&winner.worktree) {
7307                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7308                Ok(free) => {
7309                    self.state.event(
7310                        "gate",
7311                        format!(
7312                            "only {free} bytes free ({min_free} required by `[disk] \
7313                             min_free_bytes`); not spending a fix round on a failure the disk \
7314                             may explain"
7315                        ),
7316                    );
7317                    return Ok(GateFix::Stop);
7318                }
7319                Err(e) => {
7320                    self.state.event(
7321                        "gate",
7322                        format!("free disk space could not be measured ({e:#}); no fix round"),
7323                    );
7324                    return Ok(GateFix::Stop);
7325                }
7326            }
7327        }
7328
7329        let attempt = spent + 1;
7330        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7331        let base = self.landing_base();
7332        let fix_prompt = prompt::gate_fix_in(
7333            &self.state.instruction,
7334            &failed,
7335            attempt,
7336            cap,
7337            &self.state.config.graph.language,
7338            &self.state.config.graph.github_language,
7339        );
7340        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7341        let sessions = self.state.config.graph.sessions;
7342        let artifacts = self.merge_repair_artifacts();
7343        self.state.event(
7344            "gate",
7345            format!("gate failed; gate-fix round {attempt} of {cap}"),
7346        );
7347        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7348        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7349        // Persist before invoking the fixer: it may commit and then be
7350        // interrupted before its result (or the following e2e) is saved.
7351        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7352            head: before.clone(),
7353            passed: false,
7354            outcomes: Vec::new(),
7355        });
7356        self.state.save()?;
7357        let (_, seat, out) = self
7358            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7359                prompt: fix_prompt.clone(),
7360                spec,
7361                seat,
7362                cwd: winner.worktree.clone(),
7363                timeout,
7364                allow_write: true,
7365                sessions,
7366                artifacts: artifacts.clone(),
7367                stem: format!("gate-fix-{attempt}"),
7368                handover: None,
7369            })
7370            .await;
7371        // The fixer was never asked: no round is spent (nothing is recorded)
7372        // and a later reentry asks the same agent again.
7373        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7374            return Ok(GateFix::Defer);
7375        }
7376        let mut record = GateFixRecord {
7377            agent: seat.agent.clone(),
7378            failed,
7379            notes: String::new(),
7380            committed: false,
7381            error: None,
7382        };
7383        match out {
7384            AgentOutcome::Ok(o) => {
7385                // A missing report is not a failed fix: the round is judged
7386                // by the tree below, and the report only carries prose.
7387                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7388                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7389                }
7390            }
7391            AgentOutcome::Dropped(_) => {
7392                record.error = Some("the CLI dropped the stream".to_owned());
7393            }
7394            AgentOutcome::Quota(o) => {
7395                self.state.quota.push(QuotaLoss {
7396                    seat: seat.key.clone(),
7397                    node: "gate-fix".to_owned(),
7398                    at: Timestamp::now(),
7399                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7400                });
7401                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7402            }
7403            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7404        }
7405        self.state.seats.insert(seat.key.clone(), seat);
7406        let coordination = self.coordinate(&winner.worktree).await?;
7407        if let Ok(r) = git::rescue_commit(
7408            &winner.worktree,
7409            &format!("magi: gate fix {attempt} (uncommitted work)"),
7410        )
7411        .await
7412        {
7413            self.state.note_withheld("gate-fix", &r.withheld);
7414        }
7415        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7416        record.committed = after != before;
7417        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7418        let note = record.error.clone();
7419        self.state.gate_fixes.push(record);
7420        self.state.save()?;
7421        if !changed {
7422            self.state.event(
7423                "gate",
7424                match note {
7425                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7426                    None => format!("gate-fix round {attempt}: the tree did not change"),
7427                },
7428            );
7429            return Ok(GateFix::Stop);
7430        }
7431        self.state.event(
7432            "gate",
7433            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7434        );
7435
7436        drop(coordination);
7437        let wait = self.repository_wait();
7438        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7439    }
7440
7441    // ---------------------------------------------------------------- merge
7442
7443    /// One read-only rewrite by the summary's author; a text the gate still
7444    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7445    /// applies on their word or on silence. `None` means the run is parked on
7446    /// that question and nothing may be pushed or opened yet.
7447    async fn guarded_pr_message(
7448        &mut self,
7449        winner: &Candidate,
7450        facts: Option<&BranchFacts>,
7451        posting: bool,
7452    ) -> Result<Option<PrMessage>> {
7453        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7454        let head = match final_head.as_deref() {
7455            Some(h) => crate::pr_outcome::FinalHead::At(h),
7456            None => crate::pr_outcome::FinalHead::Unknown,
7457        };
7458        // Only a pull request that is about to be posted pays for a summary.
7459        let summarized = if posting {
7460            Some(self.summarize_change(winner, final_head.as_deref()).await)
7461        } else {
7462            None
7463        };
7464        // A busy shared resource is not a permanent cause: publishing an
7465        // "unavailable" description would stick (a later head move never
7466        // edits the body). Stop before anything is pushed or opened; the
7467        // run is `Blocked` and a resume asks the summary again.
7468        if self.state.resource_block.is_some() {
7469            self.state.event(
7470                "pr-summary",
7471                "a shared resource was busy; nothing was published, resume to summarize again",
7472            );
7473            return Ok(None);
7474        }
7475        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7476            anyhow::bail!(
7477                "the final commit moved during public description generation; retry before publishing"
7478            );
7479        }
7480        let note = match (&summarized, &self.state.change_summary) {
7481            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7482            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7483            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7484            (None, _) => ChangeNote::NotAsked,
7485        };
7486        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7487        if !posting {
7488            let identity = crate::scrub::Identity::current();
7489            return Ok(Some(PrMessage {
7490                title: crate::scrub::scrub(&pr.title, &identity),
7491                body: crate::scrub::scrub(&pr.body, &identity),
7492            }));
7493        }
7494        // This very text was already rejected and asked about: no second
7495        // judge call, rewrite or question, just carry the record on.
7496        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7497        if let Some(known) = self
7498            .state
7499            .github_text
7500            .clone()
7501            .filter(|g| g.fingerprint == raw_fp)
7502        {
7503            return self.settle_withheld(winner, &pr, known).await;
7504        }
7505        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7506        let mut violations =
7507            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7508        let identity = crate::scrub::Identity::current();
7509        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7510            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7511            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7512        {
7513            violations.push(crate::github_text::Violation::SensitiveData);
7514        }
7515        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7516        // only a language violation starts one; sensitive data alone goes
7517        // straight to the owner, with the redacted text as the fallback.
7518        let has_language = violations
7519            .iter()
7520            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7521        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7522            self.state.event(
7523                "github-text",
7524                if has_language {
7525                    format!("description rejected: {violations:?}; requesting one rewrite")
7526                } else {
7527                    format!("description rejected: {violations:?}; asking the owner")
7528                },
7529            );
7530            let mut rewritten = false;
7531            let mut rejected_rewrite = None;
7532            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7533            {
7534                let output = text;
7535                if let Some(title) = summary_title(&output) {
7536                    // Judge the redacted candidate that will actually be posted.
7537                    // Sensitive data alone must not discard a useful rewrite.
7538                    let raw_body = summary_without_title(&output);
7539                    let clean_title = crate::scrub::scrub(&title, &identity);
7540                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7541                    let redacted = clean_title != title || body != raw_body;
7542                    let title = clean_title;
7543                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7544                    if !complete {
7545                        self.state.event(
7546                            "github-text",
7547                            "description rewrite omitted required findings",
7548                        );
7549                    }
7550                    if !body.trim().is_empty() {
7551                        body.push_str("\n\n");
7552                        body.push_str(&public_pr_evidence(&self.state, head));
7553                    }
7554                    let marker = format!("magi:run/{}", self.state.id);
7555                    if !body.trim().is_empty() && !body.contains(&marker) {
7556                        body.push_str(&format!("\n\n{marker}"));
7557                    }
7558                    let rewrite = PrMessage {
7559                        title: title.clone(),
7560                        body: body.clone(),
7561                    };
7562                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7563                    let mut failed = crate::github_text::check_for(
7564                        &self.state.config.graph,
7565                        &title,
7566                        &body,
7567                        redecision,
7568                    );
7569                    if (!crate::github_text::shareable(&title)
7570                        || !crate::github_text::shareable(&body))
7571                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7572                    {
7573                        failed.push(crate::github_text::Violation::SensitiveData);
7574                    }
7575                    if complete
7576                        && !title.trim().is_empty()
7577                        && !body.trim().is_empty()
7578                        && !failed.is_empty()
7579                    {
7580                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7581                            title: title.clone(),
7582                            body: body.clone(),
7583                            failed: failed
7584                                .iter()
7585                                .map(|v| crate::github_text::category(*v).to_owned())
7586                                .collect(),
7587                        });
7588                    }
7589                    if complete
7590                        && !title.trim().is_empty()
7591                        && !raw_body.trim().is_empty()
7592                        && failed.is_empty()
7593                    {
7594                        decision = redecision;
7595                        pr = PrMessage { title, body };
7596                        rewritten = true;
7597                        self.state
7598                            .event("github-text", "description rewrite passed");
7599                        if redacted {
7600                            self.state.event(
7601                                "github-text",
7602                                "sensitive data removed from description rewrite",
7603                            );
7604                        }
7605                    }
7606                }
7607            }
7608            if !rewritten {
7609                self.state.event(
7610                    "github-text",
7611                    "description rewrite unavailable or rejected; asking the owner",
7612                );
7613                if let Some(w) =
7614                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7615                {
7616                    // A different text than the one asked about before: that
7617                    // question no longer describes anything.
7618                    self.retire_github_text_question("the withheld text changed");
7619                    let record = crate::run::GithubTextAsk {
7620                        rewrite: rejected_rewrite,
7621                        fingerprint: raw_fp,
7622                        title: w.title,
7623                        body: w.body,
7624                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7625                        question: None,
7626                        asks: 0,
7627                        resolved: false,
7628                        chosen_title: None,
7629                        choice: None,
7630                    };
7631                    return self.settle_withheld(winner, &pr, record).await;
7632                }
7633            }
7634        }
7635        let (title, body) =
7636            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7637        Ok(Some(PrMessage { title, body }))
7638    }
7639
7640    /// Describe the whole change from its final diff in either run mode.
7641    ///
7642    /// The summary is tied to the final commit and to the merge base of the
7643    /// resolved remote base; a record for any other pair is dropped and the
7644    /// agent asked again, so a description never outlives the diff it was
7645    /// written from. The seat is read-only and the prompt tells it to read the
7646    /// diff and the code around it. `[roles] synthesizer` selects the agents
7647    /// (unset: the default pick), each asked at most once, advancing on an
7648    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7649    /// carries the reason it is unavailable; nothing is invented then.
7650    async fn summarize_change(
7651        &mut self,
7652        winner: &Candidate,
7653        head: Option<&str>,
7654    ) -> std::result::Result<(), String> {
7655        let Some(head) = head else {
7656            return Err("the final commit could not be read".to_owned());
7657        };
7658        let remote = self.state.config.merge.remote.clone();
7659        let tracking = format!("{remote}/{}", self.state.base_branch);
7660        if !git::rev_exists(&winner.worktree, &tracking).await {
7661            return Err(format!("the remote base `{tracking}` could not be read"));
7662        }
7663        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7664            Ok(b) if !b.is_empty() => b,
7665            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7666        };
7667        let evidence = crate::pr_outcome::summary_evidence(
7668            &self.state,
7669            crate::pr_outcome::FinalHead::At(head),
7670        );
7671        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7672        if self
7673            .state
7674            .change_summary
7675            .as_ref()
7676            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7677        {
7678            return Ok(());
7679        }
7680        if self.state.change_summary.take().is_some() {
7681            self.state.event(
7682                "pr-summary",
7683                "the final diff or evidence changed; the earlier change summary was discarded",
7684            );
7685        }
7686        // Save the source snapshot even if no generation agent is available.
7687        if let Err(e) = write_artifact(
7688            &self.state,
7689            &format!("public-description-{fingerprint}.md"),
7690            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7691        ) {
7692            tracing::warn!("could not save public description evidence: {e:#}");
7693        }
7694        let chain = match agent::pick_chain(
7695            &self.state.config.agents,
7696            self.state.config.roles.synthesizer.as_ref(),
7697            &agent::installed,
7698            "synthesizer",
7699        ) {
7700            Ok(chain) => chain,
7701            Err(e) => {
7702                tracing::warn!("no agent for the change summary: {e:#}");
7703                self.state.event("pr-summary", "no summary agent available");
7704                return Err("no agent was available to summarize it".to_owned());
7705            }
7706        };
7707        let subjects = git::subjects(&winner.worktree, &base, head)
7708            .await
7709            .unwrap_or_default();
7710        let stat = git::diff_stat(&winner.worktree, &base, head)
7711            .await
7712            .unwrap_or_default();
7713        let stat: String = stat.chars().take(6_000).collect();
7714        let mut prompt = crate::prompt::change_summary_in(
7715            &base,
7716            head,
7717            &subjects,
7718            stat.trim(),
7719            &self.state.config.graph.github_language,
7720        );
7721        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7722        let sessions = self.state.config.graph.sessions;
7723        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7724        let prompts = self.state.config.prompts.clone();
7725        let cache = self.state.config.cache_dir();
7726        let run = self.state.id.clone();
7727        for (i, spec) in chain.into_iter().enumerate() {
7728            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7729            let github_spec = match agent::for_github_text(
7730                &spec,
7731                &self.state.config.graph.github_language,
7732                &winner.worktree,
7733            ) {
7734                Ok(spec) => spec,
7735                Err(e) => {
7736                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7737                    continue;
7738                }
7739            };
7740            let job = SeatJob {
7741                spec: github_spec,
7742                seat,
7743                cwd: winner.worktree.clone(),
7744                prompt: prompt.clone(),
7745                timeout,
7746                allow_write: false,
7747                sessions,
7748                artifacts: agent::artifacts_dir(&self.state.dir()),
7749                stem: if i == 0 {
7750                    "change-summary".to_owned()
7751                } else {
7752                    format!("change-summary-{}", spec.id)
7753                },
7754                handover: None,
7755            };
7756            let ctx = WaveCtx {
7757                run: &run,
7758                node: "pr-summary",
7759                prompts: &prompts,
7760                cache: cache.as_deref(),
7761                round: None,
7762                carry_seats: false,
7763                pauses: Some((&self.pause, &self.interrupt)),
7764            };
7765            let (seat, outcome) =
7766                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7767            self.state.seats.insert(seat.key.clone(), seat);
7768            let text = match outcome {
7769                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7770                AgentOutcome::ResourceBlocked(why) => {
7771                    // No agent was asked: the run stops resumably before
7772                    // anything is published (see `guarded_pr_message`).
7773                    if self.state.resource_block.is_none() {
7774                        self.state.resource_block = Some(crate::run::ResourceBlock {
7775                            cause: crate::run::ResourceBlockCause::Other,
7776                            node: "pr-summary".to_owned(),
7777                            kind: "shared resource".to_owned(),
7778                            detail: why,
7779                            at: Timestamp::now(),
7780                        });
7781                    }
7782                    return Err("a shared resource was busy".to_owned());
7783                }
7784                _ => continue,
7785            };
7786            let blind = &self.state.config.blind;
7787            let Some(title) = summary_title(&text) else {
7788                continue;
7789            };
7790            let body: String = summary_without_title(&text)
7791                .lines()
7792                .filter(|l| !l.contains("magi:run/"))
7793                .collect::<Vec<_>>()
7794                .join("\n");
7795            let body = blind::sanitize_prose(body.trim(), blind);
7796            let title = blind::sanitize_prose(&title, blind);
7797            if body.trim().is_empty() || title.trim().is_empty() {
7798                continue;
7799            }
7800            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7801                continue;
7802            }
7803            // The branch must still be the one the agent was shown.
7804            if git::rev_parse(&winner.worktree, "HEAD")
7805                .await
7806                .ok()
7807                .as_deref()
7808                != Some(head)
7809            {
7810                return Err("the branch moved while the summary was written".to_owned());
7811            }
7812            if git::merge_base(&winner.worktree, &tracking, head)
7813                .await
7814                .ok()
7815                .as_deref()
7816                != Some(base.as_str())
7817                || crate::pr_outcome::summary_evidence(
7818                    &self.state,
7819                    crate::pr_outcome::FinalHead::At(head),
7820                ) != evidence
7821            {
7822                return Err("the summary evidence changed while it was written".to_owned());
7823            }
7824            self.state.change_summary = Some(crate::run::ChangeSummary {
7825                head: head.to_owned(),
7826                base,
7827                evidence: fingerprint,
7828                title,
7829                body,
7830            });
7831            self.state.event(
7832                "pr-summary",
7833                format!("change summary written by {} for the final diff", spec.id),
7834            );
7835            if let Err(e) = self.state.save() {
7836                tracing::warn!("could not save the change summary: {e:#}");
7837            }
7838            return Ok(());
7839        }
7840        self.state
7841            .event("pr-summary", "no agent produced a change summary");
7842        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7843    }
7844
7845    /// One read-only rewrite request for a withheld description. The
7846    /// implementer's own conversation is tried first when it exists and can be
7847    /// resumed; then the `[roles] language_judge` chain (or the default pick)
7848    /// writes it in a separate conversation, skipping the implementer's agent.
7849    /// The prompt carries the whole text, so no agent needs any memory. Each
7850    /// agent is asked at most once and the chain advances on an error, quota,
7851    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
7852    /// that has run out of quota must not end the rewrite. Nothing is recorded
7853    /// as a handover.
7854    async fn rewrite_pr_text(
7855        &mut self,
7856        winner: &Candidate,
7857        pr: &PrMessage,
7858        violations: &[crate::github_text::Violation],
7859    ) -> Option<String> {
7860        let prompt = format!(
7861            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7862            pr.title, pr.body
7863        );
7864        let sessions = self.state.config.graph.sessions;
7865        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7866        // (spec, the resumed implementer's seat). A fallback seat is created
7867        // only when its turn comes, so an unused fallback leaves no record.
7868        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7869        let key = format!("impl-{}", winner.label);
7870        if let Some(spec) = self
7871            .state
7872            .config
7873            .agents
7874            .iter()
7875            .find(|a| a.id == winner.agent)
7876            .cloned()
7877            && let Some(existing) = self.state.seats.get(&key)
7878            && existing.agent == spec.id
7879            && agent::has_session(spec.kind, existing, sessions)
7880        {
7881            attempts.push((spec, Some(existing.clone())));
7882        }
7883        match agent::pick_chain(
7884            &self.state.config.agents,
7885            self.state.config.roles.language_judge.as_ref(),
7886            &agent::installed,
7887            "language_judge",
7888        ) {
7889            Ok(chain) => {
7890                for spec in chain {
7891                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7892                        continue;
7893                    }
7894                    attempts.push((spec, None));
7895                }
7896            }
7897            Err(e) if attempts.is_empty() => {
7898                tracing::warn!("no agent for the description rewrite: {e:#}");
7899                self.state
7900                    .event("github-text", "no rewrite agent available");
7901                return None;
7902            }
7903            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7904        }
7905        let prompts = self.state.config.prompts.clone();
7906        let cache = self.state.config.cache_dir();
7907        let run = self.state.id.clone();
7908        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7909            let resumed = resumed_seat.is_some();
7910            let seat = resumed_seat.unwrap_or_else(|| {
7911                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7912            });
7913            let github_spec = match agent::for_github_text(
7914                &spec,
7915                &self.state.config.graph.github_language,
7916                &winner.worktree,
7917            ) {
7918                Ok(spec) => spec,
7919                Err(e) => {
7920                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7921                    continue;
7922                }
7923            };
7924            let job = SeatJob {
7925                spec: github_spec,
7926                seat,
7927                cwd: winner.worktree.clone(),
7928                prompt: prompt.clone(),
7929                timeout: retry_budget(timeout_full, resumed),
7930                allow_write: false,
7931                sessions,
7932                artifacts: agent::artifacts_dir(&self.state.dir()),
7933                stem: if i == 0 {
7934                    "github-text-rewrite".to_owned()
7935                } else {
7936                    format!("github-text-rewrite-{}", spec.id)
7937                },
7938                handover: None,
7939            };
7940            let ctx = WaveCtx {
7941                run: &run,
7942                node: "github-text",
7943                prompts: &prompts,
7944                cache: cache.as_deref(),
7945                round: None,
7946                carry_seats: false,
7947                pauses: Some((&self.pause, &self.interrupt)),
7948            };
7949            let (seat, outcome) =
7950                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7951            self.state.seats.insert(seat.key.clone(), seat);
7952            match outcome {
7953                AgentOutcome::Ok(o)
7954                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7955                {
7956                    return Some(o.text);
7957                }
7958                AgentOutcome::ResourceBlocked(_) => return None,
7959                _ => {}
7960            }
7961        }
7962        None
7963    }
7964
7965    /// Abandon the standing posting-gate question, if any.
7966    fn retire_github_text_question(&mut self, why: &str) {
7967        let Some(id) = self
7968            .state
7969            .github_text
7970            .as_ref()
7971            .and_then(|g| g.question.clone())
7972        else {
7973            return;
7974        };
7975        let _ = ask::Questions::open().update(&id, |q| {
7976            q.abandon(why);
7977            Ok(())
7978        });
7979    }
7980
7981    /// Carry a withheld text to its end: file the question, park while it is
7982    /// open, and on an answer, silence or timeout pick the text to post. The
7983    /// decision is saved before it is used, so it applies once.
7984    async fn settle_withheld(
7985        &mut self,
7986        winner: &Candidate,
7987        pr: &PrMessage,
7988        mut g: crate::run::GithubTextAsk,
7989    ) -> Result<Option<PrMessage>> {
7990        use crate::github_text as gt;
7991        let store = ask::Questions::open();
7992        let timeout = self.state.config.graph.answer_timeout;
7993        if !g.resolved {
7994            let mut standing = g
7995                .question
7996                .as_ref()
7997                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7998            if let Some(q) = standing.as_ref()
7999                && q.status.open()
8000                && gt::expired(q, timeout)
8001            {
8002                let why = format!("no answer within {}s of asking", timeout.max(1));
8003                standing = store
8004                    .update(&q.id, |q| {
8005                        q.abandon(&why);
8006                        Ok(())
8007                    })
8008                    .ok()
8009                    .map(|(q, ())| q);
8010            }
8011            let mut ask_again: Option<Option<Vec<String>>> = None;
8012            match standing {
8013                // Recorded but never written (a stop between the two saves),
8014                // or removed: ask again without spending the retry.
8015                None if g.asks <= 1 => {
8016                    g.asks = 0;
8017                    ask_again = Some(None);
8018                }
8019                None => {
8020                    g.resolved = true;
8021                    g.chosen_title = None;
8022                    g.choice = Some(GithubTextChoice::Fallback);
8023                }
8024                Some(q) if q.status.open() => {
8025                    self.park_on_github_text(&g)?;
8026                    return Ok(None);
8027                }
8028                Some(q) => match gt::read_reply(&q) {
8029                    gt::Reply::Fallback => {
8030                        g.resolved = true;
8031                        g.chosen_title = None;
8032                        g.choice = Some(GithubTextChoice::Fallback);
8033                    }
8034                    gt::Reply::Original => {
8035                        g.resolved = true;
8036                        g.chosen_title = None;
8037                        g.choice = Some(GithubTextChoice::Original);
8038                    }
8039                    gt::Reply::Rewrite => {
8040                        g.resolved = true;
8041                        g.chosen_title = None;
8042                        g.choice = Some(if g.rewrite.is_some() {
8043                            GithubTextChoice::Rewrite
8044                        } else {
8045                            GithubTextChoice::Fallback
8046                        });
8047                    }
8048                    gt::Reply::Title(text) => {
8049                        let first = text
8050                            .lines()
8051                            .map(str::trim)
8052                            .find(|l| !l.is_empty())
8053                            .unwrap_or("");
8054                        let decision = if first.is_empty() {
8055                            None
8056                        } else {
8057                            let candidate = PrMessage {
8058                                title: first.to_owned(),
8059                                body: String::new(),
8060                            };
8061                            self.judge_pr_language(&winner.worktree, &candidate).await
8062                        };
8063                        match gt::vet_title(
8064                            &text,
8065                            decision,
8066                            self.state.config.graph.github_english(),
8067                        ) {
8068                            Ok(title) => {
8069                                g.resolved = true;
8070                                g.chosen_title = Some(title);
8071                                g.choice = Some(GithubTextChoice::MyTitle);
8072                            }
8073                            Err(cats) if g.asks < 2 => {
8074                                ask_again = Some(Some(
8075                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8076                                ))
8077                            }
8078                            Err(_) => {
8079                                g.resolved = true;
8080                                g.chosen_title = None;
8081                                g.choice = Some(GithubTextChoice::Fallback);
8082                            }
8083                        }
8084                    }
8085                },
8086            }
8087            if let Some(rejected) = ask_again {
8088                // A retry (after a rejected replacement title) is still about
8089                // the original text: the same ORIGINAL / FALLBACK comparison.
8090                let w = gt::Withheld {
8091                    title: g.title,
8092                    body: g.body,
8093                    categories: g.categories.clone(),
8094                };
8095                // Named by the original text's fingerprint alone, so a resume
8096                // rewrites the same file and the detail stays identical.
8097                let artifact = match crate::run::write_artifact(
8098                    &self.state,
8099                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8100                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8101                ) {
8102                    Ok(p) => Some(p),
8103                    Err(e) => {
8104                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8105                        None
8106                    }
8107                };
8108                let (detail, comparison) = gt::question_content(
8109                    &w,
8110                    &pr.title,
8111                    &pr.body,
8112                    &self.state.id,
8113                    rejected.as_deref(),
8114                    artifact.as_deref(),
8115                    g.rewrite.as_ref(),
8116                );
8117                let mut q = ask::Question::new(
8118                    self.state.id.clone(),
8119                    gt::ASK_NODE.to_owned(),
8120                    gt::ASK_SEAT.to_owned(),
8121                    gt::question_summary(&self.state.config.graph.language, &w),
8122                    detail,
8123                    gt::question_choices(&w, g.rewrite.as_ref()),
8124                );
8125                q.posting_comparison = Some(comparison);
8126                q.answer_timeout = timeout;
8127                // Recorded before it is written: a stop in between leaves a
8128                // record naming a question the resume then files again.
8129                g.question = Some(q.id.clone());
8130                g.asks += 1;
8131                self.state.github_text = Some(g.clone());
8132                self.state.event(
8133                    "github-text",
8134                    format!("asking the owner about the withheld text ({})", q.short()),
8135                );
8136                self.state.save()?;
8137                store
8138                    .put_executable(&mut q)
8139                    .context("file the posting-gate question")?;
8140                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8141                    tracing::warn!(
8142                        "could not notify about posting-gate question {}: {e:#}",
8143                        q.short()
8144                    );
8145                }
8146                self.park_on_github_text(&g)?;
8147                return Ok(None);
8148            }
8149        }
8150        let resolved = g.chosen_title.clone();
8151        let original = g.choice == Some(GithubTextChoice::Original);
8152        // Field names only: the event log never records the text.
8153        self.state.event(
8154            "github-text",
8155            if original {
8156                format!(
8157                    "posting the original text for: {}",
8158                    match (g.title, g.body) {
8159                        (true, true) => "title, description",
8160                        (true, false) => "title",
8161                        _ => "description",
8162                    }
8163                )
8164            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8165                "posting the saved rewrite title and description".to_owned()
8166            } else if resolved.is_some() {
8167                "posting the owner's replacement title".to_owned()
8168            } else {
8169                "posting the neutral text for the withheld fields".to_owned()
8170            },
8171        );
8172        self.state.github_text = Some(g.clone());
8173        self.state.save()?;
8174        if g.choice == Some(GithubTextChoice::Rewrite)
8175            && let Some(r) = g.rewrite.as_ref()
8176        {
8177            return Ok(Some(PrMessage {
8178                title: r.title.clone(),
8179                body: r.body.clone(),
8180            }));
8181        }
8182        let w = gt::Withheld {
8183            title: g.title,
8184            body: g.body,
8185            categories: g.categories.clone(),
8186        };
8187        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8188        if let Some(t) = resolved {
8189            title = t;
8190        }
8191        // Every withheld field is now neutral or vetted English.
8192        let english = gt::LanguageDecision {
8193            title_english: true,
8194            body_english: true,
8195            body_complete: true,
8196        };
8197        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8198        if original {
8199            // The owner's explicit choice overrides the gate for the fields it
8200            // was made for; the others keep what the gate produced.
8201            if g.title {
8202                title = pr.title.clone();
8203            }
8204            if g.body {
8205                body = pr.body.clone();
8206            }
8207        }
8208        Ok(Some(PrMessage { title, body }))
8209    }
8210
8211    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8212        self.state.github_text = Some(g.clone());
8213        self.state.end_pass_parked(ParkCause::Question);
8214        self.state.parked = true;
8215        self.state.event(
8216            "github-text",
8217            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8218        );
8219        self.state.save()
8220    }
8221
8222    /// Ask `[roles] language_judge` about this text and record which source
8223    /// decided; the text itself is never recorded.
8224    async fn judge_pr_language(
8225        &mut self,
8226        cwd: &Path,
8227        pr: &PrMessage,
8228    ) -> Option<crate::github_text::LanguageDecision> {
8229        self.state.config.roles.language_judge.as_ref()?;
8230        if !self.state.config.graph.language_gate() {
8231            return None;
8232        }
8233        let decision =
8234            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8235        self.state.event(
8236            "github-text",
8237            if decision.is_some() {
8238                "language decided by the language judge"
8239            } else {
8240                "language judge unavailable; using built-in heuristics"
8241            },
8242        );
8243        decision
8244    }
8245
8246    async fn merge(&mut self) -> Result<()> {
8247        // Same reasoning as `gate`: ask the review and gate records directly
8248        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8249        // skip can rewrite on reentry to something that no longer says
8250        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8251        // so a hand-off (open findings, green verification) reaches merge
8252        // exactly like a genuinely clean round does.
8253        //
8254        // A run resumed mid-`land` never reaches here at all: `execute`
8255        // recognises `RunStatus::Landing` before it even calls `prep`, and
8256        // routes straight to `run_land` instead. That has to happen a level
8257        // up from this function, not with a check in here, because
8258        // `review_loop`'s own status recomputation (see its doc) runs
8259        // *before* `merge` on every reentry and would otherwise overwrite
8260        // the `Landing` marker with `Gating` before this node ever saw it.
8261        if self
8262            .state
8263            .base_sync
8264            .as_ref()
8265            .is_some_and(|s| s.conflict.is_some())
8266            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8267                != Some(RunStatus::Gating)
8268            // `gate_ran == false` is not "passed" - `gate` leaves it false
8269            // both before it has ever run and when its last attempt was
8270            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8271            // permission to merge on nothing but the review record. Only a
8272            // gate that actually ran may proceed; an intentional local
8273            // review can also finish with verification explicitly not configured.
8274            // `RunState::gate_status` computes that distinction.
8275            || !self.state.gate_allows_progress()
8276        {
8277            return Ok(());
8278        }
8279        // This node's own record, not `status`: `status == Ready` is not
8280        // unique to the harmless `MergeMode::None` path this line was
8281        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8282        // run's PR was closed without merging — and on that run `mode` is
8283        // still `Pr`, so a reentry that fell through here would push and
8284        // open a second pull request. `self.state.merge` is set exactly once
8285        // this node (or `land`) has already produced a verdict, under every
8286        // mode, which is what "already done" actually means here.
8287        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8288            return Ok(());
8289        }
8290        let Some(winner) = self.state.winner().cloned() else {
8291            return Ok(());
8292        };
8293        let repo = self.state.repo.clone();
8294        let base = self.state.base_branch.clone();
8295        let mode = self.state.config.merge.mode;
8296        let style = self.state.config.merge.style;
8297        let facts = if is_review_run(&self.state) {
8298            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8299            let start = review_base(
8300                &repo,
8301                &self.state.config.merge.remote,
8302                &base,
8303                &self.state.base_commit,
8304                &winner.branch,
8305            )
8306            .await;
8307            branch_facts(&repo, &start, &winner.branch).await
8308        } else {
8309            None
8310        };
8311        // `None` when the base could not be freshly read: then an adopted
8312        // pull request's title is left alone.
8313        let leaked = if is_review_run(&self.state) {
8314            leaked_subjects(&self.state, &winner.branch).await
8315        } else {
8316            Some(Vec::new())
8317        };
8318        // `None`: parked on the owner's word about a withheld title or body.
8319        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8320        // the resume arrives here again.
8321        let Some(pr) = self
8322            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8323            .await?
8324        else {
8325            return Ok(());
8326        };
8327        let message = pr.commit_message();
8328
8329        let outcome = match mode {
8330            MergeMode::None => MergeOutcome {
8331                mode,
8332                ok: true,
8333                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8334                empty: false,
8335            },
8336            MergeMode::Pr | MergeMode::Local
8337                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8338            {
8339                MergeOutcome {
8340                    mode,
8341                    ok: false,
8342                    detail: empty_candidate_detail(&self.state, &base),
8343                    empty: true,
8344                }
8345            }
8346            MergeMode::Local => {
8347                let on = git::current_branch(&repo).await?;
8348                if on.as_deref() != Some(base.as_str()) {
8349                    MergeOutcome {
8350                        mode,
8351                        ok: false,
8352                        detail: format!(
8353                            "{} has {} checked out, not the base branch {base}",
8354                            repo.display(),
8355                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8356                        ),
8357                        empty: false,
8358                    }
8359                } else if !git::is_clean(&repo).await? {
8360                    MergeOutcome {
8361                        mode,
8362                        ok: false,
8363                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8364                        empty: false,
8365                    }
8366                } else {
8367                    let out = match style {
8368                        MergeStyle::Merge => {
8369                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8370                        }
8371                        MergeStyle::Squash => {
8372                            git::merge_squash(&repo, &winner.branch, &message).await?
8373                        }
8374                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8375                    };
8376                    MergeOutcome {
8377                        mode,
8378                        ok: out.ok(),
8379                        detail: if out.ok() { out.stdout } else { out.stderr },
8380                        empty: false,
8381                    }
8382                }
8383            }
8384            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8385                self.state.pr_publication_pending = true;
8386                self.state.save()?;
8387                let out =
8388                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8389                self.state.pr_publication_pending = out.is_err();
8390                MergeOutcome {
8391                    mode,
8392                    ok: out.is_ok(),
8393                    detail: match out {
8394                        Ok(url) => url,
8395                        Err(e) => format!("{e:#}"),
8396                    },
8397                    empty: false,
8398                }
8399            }
8400            MergeMode::Pr => {
8401                let remote = self.state.config.merge.remote.clone();
8402                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8403                if !pushed.ok() {
8404                    MergeOutcome {
8405                        mode,
8406                        ok: false,
8407                        detail: pushed.stderr,
8408                        empty: false,
8409                    }
8410                } else {
8411                    // A retry or resume of a run whose branch already has an
8412                    // open pull request adopts it rather than failing on a
8413                    // duplicate. Only this winner branch into this base:
8414                    // `branch_for` derives the name from the run id, so a
8415                    // different run's pull request never matches.
8416                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8417                    let out = match pr_merge_plan(found) {
8418                        PrPlan::Create => {
8419                            gh_pr_create(
8420                                &winner.worktree,
8421                                &base,
8422                                &winner.branch,
8423                                &pr.title,
8424                                &pr.body,
8425                            )
8426                            .await
8427                        }
8428                        PrPlan::Adopt { url, title } => {
8429                            self.state
8430                                .event("merge", format!("Pr: adopted open pull request {url}"));
8431                            if self.state.github_text.as_ref().is_some_and(|g| {
8432                                g.resolved
8433                                    && g.choice == Some(GithubTextChoice::Rewrite)
8434                                    && g.rewrite.is_some()
8435                            }) {
8436                                // The owner chose the entire pair, even if the
8437                                // existing title already matches or was edited.
8438                                land::set_pr_text_verbatim(
8439                                    &winner.worktree,
8440                                    &url,
8441                                    &pr.title,
8442                                    &pr.body,
8443                                )
8444                                .await
8445                                .map(|()| url)
8446                            } else {
8447                                if title != pr.title
8448                                    && (!is_review_run(&self.state)
8449                                        || leaked
8450                                            .as_deref()
8451                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8452                                    && let Err(e) =
8453                                        if self.state.github_text.as_ref().is_some_and(|g| {
8454                                            g.resolved
8455                                                && g.title
8456                                                && g.choice == Some(GithubTextChoice::Original)
8457                                        }) {
8458                                            // The owner chose to post this title unmodified.
8459                                            land::set_pr_title_verbatim(
8460                                                &winner.worktree,
8461                                                &url,
8462                                                &pr.title,
8463                                            )
8464                                            .await
8465                                        } else {
8466                                            land::set_pr_title(
8467                                                &mut self.state,
8468                                                &winner.worktree,
8469                                                &url,
8470                                                &pr.title,
8471                                            )
8472                                            .await
8473                                        }
8474                                {
8475                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8476                                    self.state
8477                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8478                                }
8479                                Ok(url)
8480                            }
8481                        }
8482                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8483                    };
8484                    match out {
8485                        Ok(url) => MergeOutcome {
8486                            mode,
8487                            ok: true,
8488                            detail: url,
8489                            empty: false,
8490                        },
8491                        Err(e) => MergeOutcome {
8492                            mode,
8493                            ok: false,
8494                            detail: e.to_string(),
8495                            empty: false,
8496                        },
8497                    }
8498                }
8499            }
8500        };
8501
8502        self.state.status = match (mode, outcome.ok) {
8503            (MergeMode::None, _) => RunStatus::Ready,
8504            (_, true) if outcome.empty => RunStatus::Ready,
8505            (MergeMode::Pr, true) => RunStatus::Ready,
8506            (_, true) => RunStatus::Merged,
8507            (_, false) => RunStatus::Blocked,
8508        };
8509        self.state.event(
8510            "merge",
8511            format!(
8512                "{:?}: {}",
8513                mode,
8514                outcome.detail.lines().next().unwrap_or("")
8515            ),
8516        );
8517        self.state.merge = Some(outcome);
8518        self.state.save()?;
8519
8520        // The PR is open and the run would historically stop here, leaving the
8521        // operator to watch checks, feed review comments back to a fixer, and
8522        // merge. That was done by hand six times in one session before this
8523        // existed. Opt-in, because merging is the one irreversible thing magi
8524        // can do to a repository.
8525        if self.state.config.graph.land
8526            && mode == MergeMode::Pr
8527            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8528        {
8529            self.run_land().await?;
8530        }
8531        // `run_land` may have left `status` at `Landing` - still waiting on
8532        // CI or the owner's approval, not actually settled - so this has to
8533        // read whatever `status` ended up as here, not the `Merged` this
8534        // function set a few lines up.
8535        self.settle_questions();
8536        Ok(())
8537    }
8538
8539    /// Enter `land`.
8540    ///
8541    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8542    /// a resumed run's re-entry. `land::land` itself is what serialises the
8543    /// two git-mutating moments inside the loop — the rebase push and
8544    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8545    /// to hold a lock across the whole call, and doing so would serialise
8546    /// this run's CI wait against a *different* run's land-approval resume
8547    /// in the same repository, which is exactly the "must not wait on
8548    /// another task" property the daemon's slot-freeing exists to give.
8549    async fn run_land(&mut self) -> Result<()> {
8550        crate::pr_head::ensure_owner(&self.state)?;
8551        let url = self
8552            .state
8553            .pr
8554            .as_ref()
8555            .map(|pr| pr.url.clone())
8556            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8557            .unwrap_or_default();
8558        let url = url.lines().next().unwrap_or("").trim().to_owned();
8559        if !url.starts_with("http") {
8560            return Ok(());
8561        }
8562        // A land failure is not a lost run: the work is on a branch and the
8563        // pull request is open, which is exactly where a human takes over.
8564        loop {
8565            match land::land(&mut self.state, &url).await {
8566                Ok(_)
8567                    if self
8568                        .state
8569                        .merge_repairs
8570                        .last()
8571                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8572                {
8573                    if self.repair_merge_review().await? {
8574                        continue;
8575                    }
8576                }
8577                Ok(pr) if self.state.parked => {
8578                    // `land` already saved the parked marker; nothing here
8579                    // overrides `status` back to a terminal value while an
8580                    // approval is still outstanding.
8581                    let _ = pr;
8582                }
8583                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8584                    self.state.status = RunStatus::Superseded;
8585                    self.state.save()?;
8586                }
8587                Ok(pr) => {
8588                    self.state.status = match pr.state {
8589                        land::PrLifecycle::Merged => RunStatus::Merged,
8590                        _ => RunStatus::Blocked,
8591                    };
8592                    // Downstream of a confirmed merge only - see
8593                    // `bump::should_release_bump`'s own doc for why this one
8594                    // check covers all three of `land`'s success paths.
8595                    // Best-effort: the run already landed, so a failure here
8596                    // (the decision call, `gh`, `cargo`) is recorded and never
8597                    // turns a landed run into a failed one.
8598                    if bump::should_release_bump(self.state.status)
8599                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8600                    {
8601                        // The event is the run's own record. Not-eligible cases
8602                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8603                        // `Err` is a bump that was tried and failed:
8604                        // `after_merge` itself raises the operator notice for
8605                        // that, whether or not a release PR exists yet.
8606                        self.state
8607                            .event("bump", format!("release bump skipped: {e:#}"));
8608                    }
8609                    // Independent of the bump, and best-effort in the same way:
8610                    // findings the merge left open become follow-up tasks.
8611                    if self.state.status == RunStatus::Merged {
8612                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8613                    }
8614                    self.state.save()?;
8615                }
8616                Err(e) => {
8617                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8618                        return Err(e);
8619                    }
8620                    self.state.status = RunStatus::Blocked;
8621                    self.state.event("land", format!("gave up: {e}"));
8622                    self.state.save()?;
8623                }
8624            }
8625            break;
8626        }
8627        Ok(())
8628    }
8629
8630    /// One repair cycle stays under the existing driver/task claim. An
8631    /// interrupted fixer is rolled back and held rather than run twice.
8632    async fn repair_merge_review(&mut self) -> Result<bool> {
8633        use crate::run::MergeRepairPhase as Phase;
8634        crate::pr_head::ensure_owner(&self.state)?;
8635        let i = self.state.merge_repairs.len() - 1;
8636        let request = self.state.merge_repairs[i].clone();
8637        if request.phase == Phase::Fixing {
8638            let restored = match self.state.winner().cloned() {
8639                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8640                None => Err(anyhow::anyhow!("repair has no winning branch")),
8641            };
8642            let why = match restored {
8643                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8644                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8645                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8646            };
8647            self.hold_merge_repair(i, why)?;
8648            return Ok(false);
8649        }
8650        let failure = if i >= self.state.config.graph.land_rounds
8651            || self.state.config.graph.review_rounds == 0
8652        {
8653            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8654        } else {
8655            None
8656        };
8657        if let Some(why) = failure {
8658            self.hold_merge_repair(i, why)?;
8659            return Ok(false);
8660        }
8661        if request.phase == Phase::Requested {
8662            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8663                Ok(pr) => pr,
8664                Err(e) => {
8665                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8666                    return Ok(false);
8667                }
8668            };
8669            self.state.merge_repairs[i].phase = Phase::Fixing;
8670            self.state.land_approval = None;
8671            self.state.land_armed_head = None;
8672            self.state.save()?;
8673            // Serialize the exact records, including finding ids/details,
8674            // reconsideration reasons, final votes and verification output.
8675            let evidence = serde_json::to_string_pretty(&request)?;
8676            let reason = format!(
8677                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8678            );
8679            let budget = self.state.config.graph.land_rounds;
8680            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8681            match result {
8682                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8683                Ok(land::Fixed::Failed(why)) => {
8684                    self.hold_merge_repair(
8685                        i,
8686                        format!(
8687                            "repair failed: {why}; inspect the branch and run an explicit review"
8688                        ),
8689                    )?;
8690                    return Ok(false);
8691                }
8692                Err(e) => {
8693                    self.hold_merge_repair(
8694                        i,
8695                        format!(
8696                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8697                        ),
8698                    )?;
8699                    return Ok(false);
8700                }
8701            }
8702            // Even no change gets a real new reviewer wave. The previous
8703            // evidence remains in the request; a former clean/concluded round
8704            // cannot short-circuit the new review or gate.
8705            self.begin_merge_repair_review(i)?;
8706        }
8707        self.review_loop().await?;
8708        self.gate().await?;
8709        if self.resource_halt()? || self.state.parked {
8710            return Ok(false);
8711        }
8712        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8713            != Some(RunStatus::Gating)
8714            || !self.state.gate_allows_progress()
8715        {
8716            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8717            return Ok(false);
8718        }
8719        // Review and gate may have added commits after the initial repair.
8720        // Publish exactly their final head to the existing PR, with a lease.
8721        let winner = self
8722            .state
8723            .winner()
8724            .cloned()
8725            .context("repair has no winning branch")?;
8726        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8727        if self.state.gate_head.as_deref() != Some(after.as_str())
8728            || !git::is_clean(&winner.worktree).await?
8729        {
8730            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8731            return Ok(false);
8732        }
8733        let expected = self
8734            .state
8735            .pr_pushed_head
8736            .as_deref()
8737            .unwrap_or(&request.head)
8738            .to_owned();
8739        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8740        // Push the checked commit itself: a local ref moving after the check
8741        // must not change which commit the forge receives.
8742        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8743        let refspec = format!("{after}:refs/heads/{public}");
8744        let push = git::git_raw(
8745            &winner.worktree,
8746            &["push", &lease, &self.state.config.merge.remote, &refspec],
8747        )
8748        .await;
8749        match push {
8750            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8751            Ok(out) => {
8752                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8753                return Ok(false);
8754            }
8755            Err(e) => {
8756                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8757                return Ok(false);
8758            }
8759        }
8760        self.state.merge_repairs[i].phase = Phase::Complete;
8761        self.state.status = RunStatus::Landing;
8762        self.state.event(
8763            "repair",
8764            "new review and gate completed; fresh owner merge decision required",
8765        );
8766        self.state.save()?;
8767        Ok(true)
8768    }
8769
8770    fn merge_repair_artifacts(&self) -> PathBuf {
8771        let artifacts = agent::artifacts_dir(&self.state.dir());
8772        match self
8773            .state
8774            .merge_repairs
8775            .last()
8776            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8777        {
8778            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8779            None => artifacts,
8780        }
8781    }
8782
8783    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8784        self.state.reviews.clear();
8785        self.state.gate.clear();
8786        self.state.gate_ran = false;
8787        self.state.gate_head = None;
8788        self.state.gate_fixes.clear();
8789        self.state.gate_repair_e2e = None;
8790        self.state.pre_gate.clear();
8791        self.state.pre_gate_commit = None;
8792        self.state.contested_handoff = None;
8793        self.state.land_reruns = None;
8794        self.state.land_approval = None;
8795        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8796        self.state.status = RunStatus::Reviewing;
8797        self.state.save()
8798    }
8799
8800    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8801        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8802        self.state.merge_repairs[index].failure = Some(why.clone());
8803        self.state.status = RunStatus::Blocked;
8804        self.state.event("repair", why);
8805        self.state.save()
8806    }
8807
8808    // -------------------------------------------------------------- helpers
8809
8810    /// Fetch or create a seat, keeping its conversation across nodes.
8811    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8812        if let Some(existing) = self.state.seats.get(key)
8813            && existing.agent == agent
8814        {
8815            return existing.clone();
8816        }
8817        // A seat that changes agent mints its session id from the agent too,
8818        // like a handover: the old agent's uuid is already taken by the CLI.
8819        let fresh = if self.state.seats.contains_key(key) {
8820            handover_seat(key, agent, self.state.next_seat_seed())
8821        } else {
8822            SeatState::new(key, agent, self.state.seed)
8823        };
8824        self.state.seats.insert(key.to_owned(), fresh.clone());
8825        fresh
8826    }
8827
8828    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8829    /// seat to another roster agent, in which case that agent. Nodes that
8830    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8831    /// reconsideration) must keep talking to whoever answered it, not slip
8832    /// back to the agent that failed it.
8833    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8834        match self.state.seats.get(key) {
8835            Some(s) if s.agent != spec.id => {
8836                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8837            }
8838            _ => spec,
8839        }
8840    }
8841
8842    /// A candidate rendered for judging, with the leak policy applied.
8843    fn view(&self, c: &Candidate) -> CandidateView {
8844        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8845            .unwrap_or_default();
8846        let (patch, _) = blind::sanitize_patch(
8847            &format!("candidate {} patch", c.label),
8848            &raw,
8849            &self.state.config.blind,
8850        );
8851        CandidateView {
8852            label: c.label,
8853            branch: c.branch.clone(),
8854            summary: c.summary.clone(),
8855            stat: c.stat.clone(),
8856            patch,
8857        }
8858    }
8859
8860    /// The full candidate set as prompt text, for seats with no live session.
8861    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8862        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8863        prompt::judge(
8864            "(see above)",
8865            &views,
8866            self.roles.judges.len(),
8867            base_short,
8868            "en",
8869        )
8870    }
8871
8872    /// The final-vote prompt with everything a seat that has no session of its
8873    /// own needs: the candidates, the seat's own ranking and reasons, and the
8874    /// anonymised deliberation it took part in (only when there was one, so a
8875    /// handed-over seat never sees more than the seat it replaces did). The
8876    /// `Final vote` heading stays first.
8877    fn vote_prompt_full(
8878        &self,
8879        j: usize,
8880        labels: &[char],
8881        language: &str,
8882        candidates: &[Candidate],
8883        base_short: &str,
8884    ) -> String {
8885        let mut text = format!(
8886            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8887            prompt::final_vote(labels, language),
8888            self.state.instruction,
8889            self.candidate_block(candidates, base_short)
8890        );
8891        if let Some(own) = self
8892            .state
8893            .judgements
8894            .get(j)
8895            .filter(|r| !r.ranking.is_empty())
8896        {
8897            let reasons = own
8898                .reasons
8899                .iter()
8900                .map(|(k, v)| format!("- {k}: {v}"))
8901                .collect::<Vec<_>>()
8902                .join("\n");
8903            text.push_str(&format!(
8904                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8905                own.ranking.iter().collect::<String>(),
8906                if reasons.is_empty() {
8907                    ""
8908                } else {
8909                    ", because:\n"
8910                }
8911            ));
8912        }
8913        if !self.state.deliberation.is_empty() {
8914            text.push_str("\n# What was argued before this vote\n");
8915            for t in self.transcript(&[], j) {
8916                text.push_str(&format!(
8917                    "\n## {}{}\n\n{}\n",
8918                    t.who,
8919                    if t.is_self { " (you)" } else { "" },
8920                    t.body.trim()
8921                ));
8922            }
8923        }
8924        text
8925    }
8926
8927    /// Anonymised transcript for judge `self_idx`.
8928    ///
8929    /// The initial rankings are always the opening statements. Seeding them
8930    /// only when no turn had been taken yet meant every judge after the first
8931    /// argued against a single voice instead of against the actual split — the
8932    /// disagreement is the information, so it is always on the table.
8933    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8934        let mut turns = Vec::new();
8935        for j in &self.state.judgements {
8936            if j.ranking.is_empty() {
8937                continue;
8938            }
8939            let reasons = j
8940                .reasons
8941                .iter()
8942                .map(|(k, v)| format!("- {k}: {v}"))
8943                .collect::<Vec<_>>()
8944                .join("\n");
8945            turns.push(Turn {
8946                who: format!("Judge {} (opening ranking)", j.judge),
8947                is_self: j.judge == self_idx + 1,
8948                body: format!(
8949                    "Ranked {}{}{reasons}",
8950                    j.ranking.iter().collect::<String>(),
8951                    if reasons.is_empty() {
8952                        ""
8953                    } else {
8954                        ", because:\n"
8955                    }
8956                ),
8957            });
8958        }
8959        for t in self
8960            .state
8961            .deliberation
8962            .iter()
8963            .flat_map(|r| r.turns.iter())
8964            .chain(current)
8965        {
8966            turns.push(Turn {
8967                who: format!("Judge {}", t.judge),
8968                is_self: t.judge == self_idx + 1,
8969                body: t.body.clone(),
8970            });
8971        }
8972        turns
8973    }
8974}
8975
8976/// Does this seat still hold the context a follow-up prompt would rely on?
8977fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8978    agent::has_session(spec.kind, seat, sessions)
8979}
8980
8981/// The next entry in `roster` after `start`, never wrapping back to the
8982/// front, whose id is not in `tried` yet.
8983///
8984/// Starts one past `start` rather than at the front of `roster`: `start` is
8985/// the seat's own original position, and a seat whose candidate slot already
8986/// sits on the roster's second entry must fall through to the third next, not
8987/// restart at the first — which is very likely a different candidate's own
8988/// agent already. Never wraps back past `start`, for the same reason: an
8989/// entry earlier in the roster than the seat's own position is almost
8990/// certainly some *other* candidate slot's own agent, and once the tail of
8991/// the roster is exhausted there are no more untried agents for *this* seat
8992/// to fall through to — the caller's fallback chain ends there, exactly as
8993/// "no further untried agents remain in the list for that seat" asks for.
8994///
8995/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8996/// the same id twice (an operator's `roles.implementers` typo, or a
8997/// `[[agents]]` list reused across roles) must not let
8998/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8999/// over `roster` either finds an untried id or runs out, so this always
9000/// terminates regardless of duplicates.
9001fn next_untried_in_roster<'a>(
9002    roster: &'a [AgentSpec],
9003    start: usize,
9004    tried: &BTreeSet<String>,
9005) -> Option<&'a AgentSpec> {
9006    roster
9007        .get(start + 1..)?
9008        .iter()
9009        .find(|s| !tried.contains(&s.id))
9010}
9011
9012/// The successor for a seat, shared by all four seat kinds (implement, judge,
9013/// review, advise). `others` holds the ids that *currently* occupy the other
9014/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9015/// sees them), so roster entries beyond the seat count act as spares: a failed
9016/// seat goes to an agent no other seat holds whenever the roster permits.
9017///
9018/// `carried` is `Some` only for the review loop's carried failure history.
9019/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9020/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9021/// whole roster: untried and not another seat's occupant; (3) the plain walk
9022/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9023/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9024/// a better outcome than an empty seat, and it keeps the answer to "is there
9025/// a successor at all" exactly what it was before occupants were considered,
9026/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9027/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9028/// carried failure is a cheaper bet than doubling an agent on the panel, and
9029/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9030/// the same round are handled one at a time, so a seat later in the batch
9031/// sees an earlier one's new occupant but not yet its own freed agent.
9032fn pick_successor<'a>(
9033    roster: &'a [AgentSpec],
9034    start: usize,
9035    tried: &BTreeSet<String>,
9036    carried: Option<&BTreeSet<String>>,
9037    others: &BTreeSet<String>,
9038) -> Option<&'a AgentSpec> {
9039    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9040    roster
9041        .get(start + 1..)
9042        .and_then(|tail| {
9043            tail.iter()
9044                .filter(free)
9045                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9046        })
9047        .or_else(|| {
9048            carried
9049                .is_some()
9050                .then(|| roster.iter().find(free))
9051                .flatten()
9052        })
9053        .or_else(|| match carried {
9054            Some(c) => next_for_seat(roster, start, tried, c),
9055            None => next_untried_in_roster(roster, start, tried),
9056        })
9057}
9058
9059/// The next agent for a seat that carries its failure history across rounds
9060/// (the review loop). `round_tried` is this round's own bound and starts
9061/// empty every round; `carried_failed` only decides priority.
9062///
9063/// First: an id walking forward from `start`, never wrapping, that is neither
9064/// tried this round nor failed in an earlier one. Only when that is exhausted
9065/// does it rescue: the first roster id (in roster order, so this one *does*
9066/// look before `start`) not yet tried this round, which is by then a carried
9067/// failure. Each id is rescued at most once per round, so it cannot loop.
9068fn next_for_seat<'a>(
9069    roster: &'a [AgentSpec],
9070    start: usize,
9071    round_tried: &BTreeSet<String>,
9072    carried_failed: &BTreeSet<String>,
9073) -> Option<&'a AgentSpec> {
9074    roster
9075        .get(start + 1..)?
9076        .iter()
9077        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9078        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9079}
9080
9081/// Where a reviewer seat starts a round: the agent that last answered it when
9082/// it is still on the roster and not marked failed, else the spec's own agent
9083/// unless it failed, else the next roster agent that has not failed, else the
9084/// spec's own agent again (the whole roster failed). Ids no longer on the
9085/// roster are ignored. An empty roster has no handover, so the spec stands.
9086fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9087    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9088        return spec;
9089    };
9090    let ok = |id: &str| !h.failed.contains(id);
9091    if let Some(last) = h.last_ok.as_deref()
9092        && ok(last)
9093        && let Some(s) = roster.iter().find(|s| s.id == last)
9094    {
9095        return s.clone();
9096    }
9097    if ok(&spec.id) {
9098        return spec;
9099    }
9100    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9101    roster
9102        .get(from + 1..)
9103        .into_iter()
9104        .flatten()
9105        .chain(roster.iter())
9106        .find(|s| ok(&s.id))
9107        .cloned()
9108        .unwrap_or(spec)
9109}
9110
9111/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9112/// seed so a CLI that mints its session id up front (`--session-id`) never
9113/// reuses the uuid the previous agent already opened under the same seat key.
9114pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9115    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9116}
9117
9118/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9119/// so the classifier below and the code that builds the message cannot drift.
9120const TIMED_OUT: &str = "timed out";
9121
9122impl FailClass {
9123    /// `None` for an answer; otherwise how the turn failed.
9124    fn of(out: &AgentOutcome) -> Option<Self> {
9125        match out {
9126            AgentOutcome::Ok(_) => None,
9127            AgentOutcome::Quota(_) => Some(Self::Quota),
9128            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9129            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9130            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9131            // Not a failure of the agent. Callers check for it first; `None`
9132            // here means "no class to hand over on".
9133            AgentOutcome::ResourceBlocked(_) => None,
9134        }
9135    }
9136
9137    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9138    fn stem_word(&self) -> &'static str {
9139        match self {
9140            Self::Quota => "quota",
9141            _ => "handover",
9142        }
9143    }
9144}
9145
9146/// The message's first line with its variable parts removed — digit runs and
9147/// path-like tokens — so "exit code 2" and "exit code 7" read
9148/// as one kind of failure.
9149fn failure_signature(msg: &str) -> String {
9150    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9151    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9152    // the historical machine key while the process diagnosis uses readable prose.
9153    if let Some(code) = line
9154        .strip_prefix("process exited with no usable output (exit code ")
9155        .and_then(|s| s.strip_suffix(')'))
9156        .and_then(|s| s.parse::<i32>().ok())
9157    {
9158        line = format!("exited with some({code}) and no usable output");
9159    } else if matches!(
9160        line.as_str(),
9161        "process exited with no usable output (no exit code reported)"
9162            | "process exited with no usable output (exit code unavailable)"
9163    ) {
9164        line = "exited with none and no usable output".to_owned();
9165    }
9166    let mut out = Vec::new();
9167    for word in line.split_whitespace() {
9168        if word.contains('/') || word.contains('\\') {
9169            out.push("<path>".to_owned());
9170            continue;
9171        }
9172        let mut w = String::new();
9173        let mut in_digits = false;
9174        for c in word.chars() {
9175            if c.is_ascii_digit() {
9176                if !in_digits {
9177                    w.push('#');
9178                }
9179                in_digits = true;
9180            } else {
9181                in_digits = false;
9182                w.push(c);
9183            }
9184        }
9185        out.push(w);
9186    }
9187    out.join(" ").chars().take(120).collect()
9188}
9189
9190/// Whether a seat that just failed with `cur` may go to the next roster agent.
9191/// A quota or a timeout always may. Any other failure may not when the agent
9192/// before it failed the same way: an error the prompt causes would otherwise
9193/// walk the whole roster. `prev` is the class of the immediately preceding
9194/// agent's failure, so a quota or timeout in between breaks the run of
9195/// identical failures by itself.
9196fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9197    match cur {
9198        FailClass::Quota | FailClass::Timeout => true,
9199        FailClass::Other(_) => prev != Some(cur),
9200    }
9201}
9202
9203/// A short human reason for a failed outcome, for the handover record.
9204fn fail_reason(out: &AgentOutcome) -> String {
9205    match out {
9206        AgentOutcome::Ok(_) => String::new(),
9207        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9208        AgentOutcome::Dropped(o) => format!(
9209            "the CLI dropped the stream ({})",
9210            o.dropped
9211                .as_ref()
9212                .map(|d| d.why.as_str())
9213                .unwrap_or("it ended without delivering its answer")
9214        ),
9215        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9216            e.lines().next().unwrap_or("").chars().take(160).collect()
9217        }
9218    }
9219}
9220
9221/// Note one handover in the run: the structured record and, in the timeline,
9222/// the sentence a person reads. A quota keeps the wording it always had.
9223pub(crate) fn record_handover(
9224    state: &mut RunState,
9225    node: &str,
9226    seat: &str,
9227    from: &str,
9228    to: &str,
9229    class: &FailClass,
9230    reason: &str,
9231) {
9232    let message = if *class == FailClass::Quota {
9233        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9234    } else {
9235        format!("{seat}: handed over {from} -> {to} ({reason})")
9236    };
9237    state.event(node, message);
9238    state.handovers.push(Handover {
9239        at: Timestamp::now(),
9240        node: node.to_owned(),
9241        seat: seat.to_owned(),
9242        from: from.to_owned(),
9243        to: to.to_owned(),
9244        reason: reason.to_owned(),
9245    });
9246}
9247
9248/// Did this reply report running a command whose own CLI never confirmed an
9249/// exit status?
9250///
9251/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9252/// command *finished* (see that type's own doc), so this can only be `true`
9253/// for a command whose completion event carried no readable exit code — not
9254/// for one that simply is not mentioned at all. That is the one signal this
9255/// crate can read, from the same record `state.jobs` renders, about a reply
9256/// standing next to work its own CLI cannot vouch for finishing; it is
9257/// deliberately not a check on the exit code's *value* (a fixer legitimately
9258/// runs a command that fails mid-iteration before it succeeds) and not a
9259/// guess at a command still running in the background (which emits no event
9260/// at all, and so leaves no evidence here to find).
9261fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9262    commands.iter().any(|c| c.exit_code.is_none())
9263}
9264
9265/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9266/// trusted as a verified no-op — the adoption guard's own text-level half.
9267///
9268/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9269/// timed out): a marker only earns the benefit of the doubt from a turn the
9270/// CLI itself vouches for finishing properly, the same house style
9271/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9272/// *fix* report to for `commands`. A candidate that timed out, exited
9273/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9274/// is, whatever prose it wrote — this returns `None` before it ever looks at
9275/// `text`. The remaining guards (the tree really is empty, the evidence is
9276/// non-empty) are the caller's: this only reads what the reply *claimed*.
9277fn verified_noop_claim(
9278    usable: bool,
9279    commands: &[agent::CommandEvidence],
9280    text: &str,
9281) -> Option<String> {
9282    (usable && !has_unconfirmed_command(commands))
9283        .then(|| verdict::verified_noop(text))
9284        .flatten()
9285}
9286
9287fn short(commit: &str) -> String {
9288    commit.chars().take(7).collect()
9289}
9290
9291fn make_executable(path: &Path) -> Result<()> {
9292    #[cfg(unix)]
9293    {
9294        use std::os::unix::fs::PermissionsExt as _;
9295        let mut perms = std::fs::metadata(path)?.permissions();
9296        perms.set_mode(0o755);
9297        std::fs::set_permissions(path, perms)?;
9298    }
9299    #[cfg(not(unix))]
9300    {
9301        let _ = path;
9302    }
9303    Ok(())
9304}
9305
9306/// What every seat in one batch shares: where the answers are attributed, the
9307/// prompt overlay they inherit, and the build cache they are told to use.
9308///
9309/// A struct rather than four more parameters: `wave` also needs the run's
9310/// state (to record who is answering right now) and the attempt number, and
9311/// eight positional arguments is both unreadable and a clippy error.
9312struct WaveCtx<'a> {
9313    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9314    /// paid for it.
9315    run: &'a str,
9316    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9317    node: &'a str,
9318    prompts: &'a Prompts,
9319    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9320    cache: Option<&'a Path>,
9321    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9322    /// `JobRecord::round`. `None` for every other node.
9323    round: Option<usize>,
9324    /// Carry each seat's failed-agent history across waves (the review loop
9325    /// only): start-of-round priority and handover choice read
9326    /// [`RunState::seat_history`], and every answer or failure writes it.
9327    carry_seats: bool,
9328    /// The run's park / interrupt requests, so a wait for a shared resource
9329    /// can be abandoned the moment the operator asks to park.
9330    pauses: Option<(&'a Pause, &'a Pause)>,
9331}
9332
9333/// Shared scheduling policy for operations and spawned seat invocations.
9334fn repository_wait(
9335    state: &RunState,
9336    pauses: Option<(&Pause, &Pause)>,
9337) -> crate::repo_lock::WaitPolicy {
9338    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9339    let inherited = crate::repo_lock::WaitPolicy::current();
9340    crate::repo_lock::WaitPolicy::new(
9341        Duration::from_secs(state.config.graph.timeout_resource_wait),
9342        move || {
9343            inherited.cancelled()
9344                || pauses
9345                    .as_ref()
9346                    .is_some_and(|(a, b)| a.parked() || b.parked())
9347        },
9348    )
9349}
9350
9351fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9352    if error.is::<crate::repo_lock::ResourceBlocked>() {
9353        state.resource_block = Some(crate::run::ResourceBlock {
9354            cause: crate::run::ResourceBlockCause::Other,
9355            node: "repository".into(),
9356            kind: if error.to_string().contains("disk pressure") {
9357                "disk pressure"
9358            } else {
9359                "repository coordination"
9360            }
9361            .into(),
9362            detail: error.to_string(),
9363            at: Timestamp::now(),
9364        });
9365        state.event("resource", error.to_string());
9366    }
9367}
9368
9369async fn coordinate_operation(
9370    state: &mut RunState,
9371    cwd: &Path,
9372    wait: &crate::repo_lock::WaitPolicy,
9373) -> Result<Option<crate::repo_lock::SeatLease>> {
9374    let result =
9375        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9376            .await;
9377    if let Err(e) = &result {
9378        record_repository_block(state, e);
9379    }
9380    result
9381}
9382
9383/// Run one job, honouring the parallelism budget.
9384async fn run_one(
9385    job: SeatJob,
9386    sem: Arc<Semaphore>,
9387    ctx: &WaveCtx<'_>,
9388    state: &mut RunState,
9389    attempt: usize,
9390) -> (SeatState, AgentOutcome) {
9391    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9392        .await
9393        .pop()
9394        .expect("one job in, one result out");
9395    (seat, out)
9396}
9397
9398/// Run every job concurrently, capped by the semaphore, preserving order.
9399///
9400/// Every seat in the batch is recorded into [`RunState::active`] before the
9401/// wave starts and cleared as each answer lands, so the run's own record says
9402/// who is still being waited on rather than only who finished.
9403async fn wave(
9404    jobs: Vec<SeatJob>,
9405    sem: Arc<Semaphore>,
9406    ctx: &WaveCtx<'_>,
9407    state: &mut RunState,
9408    attempt: usize,
9409) -> Vec<(usize, SeatState, AgentOutcome)> {
9410    let WaveCtx {
9411        run,
9412        node,
9413        prompts,
9414        cache,
9415        round,
9416        carry_seats: _,
9417        pauses,
9418    } = *ctx;
9419    for job in &jobs {
9420        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9421    }
9422    if let Err(e) = state.save() {
9423        // A failed persist of "who is answering right now" must not abort the
9424        // wave: the seats are already being asked, and the alternative is
9425        // losing the answers to save a status line nobody may even be
9426        // watching.
9427        tracing::warn!("could not persist in-progress seats: {e:#}");
9428    }
9429    // Hold the shared build cache's lease for the whole batch, not per job:
9430    // several candidates (an implement wave) or a fixer legitimately share
9431    // one cache concurrently within this run, and that stays untouched — a
9432    // single lease taken once for the whole wave and released once it is
9433    // done is what stops a *different* borrower (another run's own wave, its
9434    // e2e/gate, a human's `magi review`) from interleaving a build into the
9435    // same directory while this one is in flight. Best-effort, not
9436    // all-or-nothing: a wave that cannot get the lease within its own
9437    // longest job's budget still runs — an hour of paid implementer calls is
9438    // not thrown away over cache contention — but every write-allowed seat
9439    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9440    // below), the same fallback a read-only seat always gets, rather than
9441    // building into a directory this run was never granted. The identity
9442    // record is still invalidated below either way, so the next tracked
9443    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9444    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9445    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9446    // never borrowed from a job's agent budget (and never charged to it either,
9447    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9448    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9449    let cache_guard = if let Some(cache_dir) = cache {
9450        if jobs_had_a_writer {
9451            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9452            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9453            match acquire_cache_lease(
9454                state,
9455                cache_dir,
9456                &owner,
9457                resource_limit,
9458                node,
9459                node,
9460                &cancel,
9461            )
9462            .await
9463            {
9464                Ok(Some(g)) => Some(g),
9465                // Parking: leave without spawning anything. The run parks at
9466                // the next boundary and the seats are asked afresh on resume.
9467                Ok(None) => {
9468                    if let Some(block) = &mut state.resource_block {
9469                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9470                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9471                        };
9472                    }
9473                    return block_wave(
9474                        jobs,
9475                        state,
9476                        node,
9477                        attempt,
9478                        "parked while waiting for the shared build cache",
9479                    );
9480                }
9481                // An unreadable lease is not contention: keep the old
9482                // fallback (no shared cache for this wave's seats).
9483                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9484                Err(e) => {
9485                    // Writing seats may not build into a cache this run does
9486                    // not hold, and an agent that cannot be given one is not
9487                    // asked at all: the run stops resumably instead.
9488                    let why = format!("{e:#}");
9489                    state.event(
9490                        "resource",
9491                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9492                    );
9493                    return block_wave(jobs, state, node, attempt, &why);
9494                }
9495            }
9496        } else {
9497            None
9498        }
9499    } else {
9500        None
9501    };
9502    // Every job keeps its whole budget: the wait above was not the agent's.
9503    // Restart the seats' clocks so "started" means "being spawned now".
9504    state.seats_spawning(node, attempt);
9505    if let Err(e) = state.save() {
9506        tracing::warn!("could not persist the start of a wave: {e:#}");
9507    }
9508    // Queueing behind this wave's own seats for a parallelism slot is normal
9509    // (each of them is bounded by its own agent timeout); only a wait beyond
9510    // all of them plus the resource limit means the slots never came back.
9511    let slot_limit = resource_limit
9512        + jobs
9513            .iter()
9514            .map(|j| j.timeout)
9515            .max()
9516            .unwrap_or_default()
9517            .saturating_mul(jobs.len() as u32);
9518    let mut set = tokio::task::JoinSet::new();
9519    // Seats announce themselves here the moment they hold a parallelism
9520    // permit, i.e. just before their CLI is spawned.
9521    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9522    let overlay = prompts.overlay(node);
9523    let repository_wait = repository_wait(state, pauses);
9524    for (i, mut job) in jobs.into_iter().enumerate() {
9525        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9526        if cache.is_some() {
9527            job.prompt.push('\n');
9528            job.prompt
9529                .push_str(&prompt::build_cache_note(node, job.allow_write));
9530        }
9531        let sem = Arc::clone(&sem);
9532        let run = run.to_owned();
9533        let node = node.to_owned();
9534        // Only implementers were told about the task's attachments, so only
9535        // their seats get the directory widened for reading.
9536        let attachments = if node == "implement" {
9537            state.attachments.clone()
9538        } else {
9539            Vec::new()
9540        };
9541        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9542        // `prompt::build_cache_note`'s doc for why setting it anyway is
9543        // exactly how a sandboxed reviewer's write refusal got reported as a
9544        // defect in the patch, not a property of its own seat. And a
9545        // write-allowed one is handed it only when the lease above was
9546        // actually acquired: a wave that could not get it (`cache_guard` is
9547        // `None`, see its own comment) must not send seats to build into a
9548        // directory this run does not hold - that is the exact concurrent,
9549        // unmanaged-write race this module exists to prevent, not something
9550        // "proceeding anyway" is allowed to reintroduce.
9551        let cache = cache
9552            .filter(|_| job.allow_write && cache_guard.is_some())
9553            .map(Path::to_path_buf);
9554        let spawned_tx = spawned_tx.clone();
9555        let repository_wait = repository_wait.clone();
9556        let coordination = state.config.coordination.clone();
9557        let fetched_base = state.fetched_base.clone();
9558        set.spawn(async move {
9559            // A wait for a parallelism slot is scheduling, not the agent: it
9560            // is bounded by its own limit and ends as a resource block, never
9561            // as a timeout of the agent that was not started.
9562            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9563                Ok(p) => p,
9564                Err(_) => {
9565                    return (
9566                        i,
9567                        job.seat,
9568                        AgentOutcome::ResourceBlocked(format!(
9569                            "no parallel slot became free within {}s",
9570                            slot_limit.as_secs()
9571                        )),
9572                        false,
9573                    );
9574                }
9575            };
9576            spawned_tx.send(job.seat.key.clone()).ok();
9577            let mut seat = job.seat;
9578            let out = repository_wait
9579                .scope(agent::invoke(
9580                    &job.spec,
9581                    &mut seat,
9582                    &Invocation {
9583                        scope: crate::agent::InvocationScope::Repository,
9584                        coordination: coordination.as_ref(),
9585                        fetched_base: fetched_base.as_ref(),
9586                        cwd: &job.cwd,
9587                        prompt: &job.prompt,
9588                        timeout: job.timeout,
9589                        allow_write: job.allow_write,
9590                        unsandboxed: false,
9591                        sessions: job.sessions,
9592                        artifacts: &job.artifacts,
9593                        stem: &job.stem,
9594                        run: &run,
9595                        node: &node,
9596                        cache_dir: cache.as_deref(),
9597                        attachments: &attachments,
9598                        writable: &[],
9599                    },
9600                ))
9601                .await;
9602            let interrupted = out
9603                .as_ref()
9604                .err()
9605                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9606            let out = match out {
9607                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9608                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9609                // Billed work the CLI failed to hand over is not an ordinary
9610                // failure, but its text is the CLI's raw error JSON, not an
9611                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9612                // read it as one by forgetting to check. `usable()` is always
9613                // false here (dropped implies an empty response), so this has
9614                // to be checked before the catch-all `Failed` below or the
9615                // one shape this exists for is lost with the rest.
9616                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9617                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9618                Ok(o) => AgentOutcome::Failed(format!(
9619                    "process exited with no usable output ({})",
9620                    crate::notices::exit_status(o.exit_code)
9621                )),
9622                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9623                    AgentOutcome::ResourceBlocked(e.to_string())
9624                }
9625                Err(e) => AgentOutcome::Failed(e.to_string()),
9626            };
9627            (i, seat, out, interrupted)
9628        });
9629    }
9630    drop(spawned_tx);
9631    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9632    loop {
9633        let joined = tokio::select! {
9634            Some(key) = spawned_rx.recv() => {
9635                // The clock of this seat starts now, not when it queued.
9636                state.seat_spawned(&key);
9637                if let Err(e) = state.save() {
9638                    tracing::warn!("could not persist a seat's start: {e:#}");
9639                }
9640                continue;
9641            }
9642            joined = set.join_next() => joined,
9643        };
9644        let Some(joined) = joined else {
9645            break;
9646        };
9647        let (i, seat, out, interrupted) = match joined {
9648            Ok(v) => v,
9649            // No seat to clear: a panicked task never reported which one it
9650            // was. The defensive sweep below this loop is what stops that
9651            // seat's `active` entry from surviving forever.
9652            Err(e) => {
9653                tracing::error!("agent task panicked: {e}");
9654                continue;
9655            }
9656        };
9657        state.seat_finished(&seat.key);
9658        if interrupted
9659            && node == "implement"
9660            && let Some(label) = state
9661                .candidates
9662                .iter()
9663                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9664                .map(|candidate| candidate.label)
9665        {
9666            state.candidate_preservation.insert(
9667                label,
9668                "interrupted implementation awaiting persistence".into(),
9669            );
9670            state.interrupted_implementations.insert(label);
9671        }
9672        if let AgentOutcome::ResourceBlocked(why) = &out
9673            && state.resource_block.is_none()
9674        {
9675            state.resource_block = Some(crate::run::ResourceBlock {
9676                cause: crate::run::ResourceBlockCause::Other,
9677                node: node.to_owned(),
9678                kind: if why.contains("disk pressure") {
9679                    "disk pressure"
9680                } else if why.contains("repository coordination") {
9681                    "repository coordination"
9682                } else {
9683                    "parallel slot"
9684                }
9685                .to_owned(),
9686                detail: why.clone(),
9687                at: Timestamp::now(),
9688            });
9689            let activity = if interrupted {
9690                "agent execution interrupted"
9691            } else {
9692                "no agent was started"
9693            };
9694            state.event("resource", format!("{node}: {activity} - {why}"));
9695        }
9696        record_jobs(state, node, round, &seat.key, &out);
9697        if let Err(e) = state.save() {
9698            tracing::warn!("could not persist a seat's completion: {e:#}");
9699        }
9700        if collected.len() <= i {
9701            collected.resize_with(i + 1, || None);
9702        }
9703        collected[i] = Some((i, seat, out));
9704    }
9705    // Belt-and-braces for the panic branch above: every seat this exact batch
9706    // started shares this `(node, attempt)` pair, and every seat that finished
9707    // normally already cleared itself, so anything left tagged with it here
9708    // can only be a panicked task's leftover. Cleared unconditionally rather
9709    // than left to read as still answering forever.
9710    if state
9711        .active
9712        .values()
9713        .any(|a| a.node == node && a.attempt == attempt)
9714    {
9715        state
9716            .active
9717            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9718        if let Err(e) = state.save() {
9719            tracing::warn!("could not persist the end of a wave: {e:#}");
9720        }
9721    }
9722    // Whether or not the lease above was actually held, several worktrees
9723    // may just have built into the cache with nothing here able to name one
9724    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9725    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9726    // clean it might not have strictly needed; trusting a stale match would
9727    // cost it a wrong answer.
9728    if let Some(cache_dir) = cache
9729        && jobs_had_a_writer
9730    {
9731        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9732    }
9733    if let Some(guard) = cache_guard {
9734        guard.release();
9735    }
9736    collected.into_iter().flatten().collect()
9737}
9738
9739/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9740/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9741/// this is data collection, not the fix-specific completion contract in
9742/// [`Runner::continue_fix_report`], and applies regardless of which node
9743/// asked.
9744///
9745/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9746/// evidence from; `Failed` does not, and correctly contributes nothing — a
9747/// timeout or crash is not itself evidence about a command the seat may have
9748/// started.
9749fn record_jobs(
9750    state: &mut RunState,
9751    node: &str,
9752    round: Option<usize>,
9753    seat: &str,
9754    out: &AgentOutcome,
9755) {
9756    let commands: &[agent::CommandEvidence] = match out {
9757        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9758        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9759    };
9760    let checked_at = Timestamp::now();
9761    for c in commands {
9762        state.jobs.push(JobRecord {
9763            node: node.to_owned(),
9764            round,
9765            seat: seat.to_owned(),
9766            id: c.id.clone(),
9767            description: c.description.clone(),
9768            checked_at,
9769            status: match c.exit_code {
9770                Some(0) => JobStatus::Completed,
9771                Some(_) => JobStatus::Failed,
9772                None => JobStatus::Unknown,
9773            },
9774            exit_code: c.exit_code,
9775            result_summary: c.result_summary.clone(),
9776            source: c.source.clone(),
9777        });
9778    }
9779}
9780
9781/// Is a review round clean, given how many reviewer seats answered against
9782/// how many the round expected?
9783///
9784/// A seat that never answered (timeout, crash, unparsable output) is not a
9785/// seat that read the patch and found nothing — treating it as such is
9786/// exactly the bug this function exists to close. Under the default `block`
9787/// policy a missing seat can never be clean; `warn` still requires the seats
9788/// that *did* answer to have found nothing blocking and verification to be
9789/// green.
9790///
9791/// `quota_missing` narrows that `block` default for exactly one cause of
9792/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9793/// a session limit lifts by the very next round buys nothing — the seat is
9794/// asked again with the same quota — so once every missing seat is accounted
9795/// for by a quota loss (and at least one seat *did* answer, so a decision has
9796/// something to rest on) the round is decided on the panel that could answer,
9797/// same as `warn` would. A panel that lost every seat to quota is not
9798/// decided here: `answered == 0` falls through to the existing `block`
9799/// fallback so a fully collapsed panel still waits rather than landing on no
9800/// review at all.
9801fn round_is_clean(
9802    blocking: usize,
9803    e2e_ok: bool,
9804    answered: usize,
9805    expected: usize,
9806    quota_missing: usize,
9807    policy: IncompleteReviewPolicy,
9808) -> bool {
9809    if blocking != 0 || !e2e_ok {
9810        return false;
9811    }
9812    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9813        return true;
9814    }
9815    answered > 0 && expected - answered <= quota_missing
9816}
9817
9818/// The review loop's own conclusion, derived entirely from its persisted
9819/// round records and the round budget that produced them — never from
9820/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9821/// recomputes the identical answer regardless of what an earlier node in the
9822/// same walk, or a previous walk, did to `status`.
9823///
9824/// `None` while more rounds remain to try, including when review never ran
9825/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9826/// gone clean, or the budget is spent, or the tree has stopped moving (see
9827/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9828///
9829/// - An incomplete panel that raised nothing is missing input, not a
9830///   verified tree — never a hand-off candidate, whatever verification said
9831///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9832/// - Otherwise, green e2e on the last round hands off (see
9833///   [`Runner::stop_reviewing`]); red e2e blocks.
9834///
9835/// A last round whose own verification is still `ResourceBlocked` — magi
9836/// itself never got a command to run, not evidence the patch is broken —
9837/// is neither: this returns `None` for it too, the same as "more rounds
9838/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9839/// handling of that shape) instead of this cheap recomputation guessing a
9840/// verdict a real attempt never produced.
9841pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9842    if max_rounds == 0
9843        || reviews.last().is_some_and(|r| {
9844            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9845        })
9846    {
9847        return Some(RunStatus::Gating);
9848    }
9849    let last = reviews.last()?;
9850    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9851    if reviews.len() < max_rounds && !stagnant {
9852        return None;
9853    }
9854    if last.incomplete() && last.blocking == 0 {
9855        return Some(RunStatus::Blocked);
9856    }
9857    if last.e2e_status() == E2eStatus::ResourceBlocked
9858        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9859    {
9860        return None;
9861    }
9862    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9863        RunStatus::Gating
9864    } else {
9865        RunStatus::Blocked
9866    })
9867}
9868
9869/// How long a re-ask may take, given the budget the first attempt had.
9870///
9871/// A `nudged` retry is a request to restate an answer the seat has already
9872/// worked out: it carries no new work, so it does not deserve the original
9873/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9874/// 133 seconds while a third sat for over ten minutes on a resumed session
9875/// holding 410 KB of prior output - and because the retry had inherited the
9876/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9877/// judging round whose other seats were long finished.
9878///
9879/// A quarter of the budget, with a floor so that a deliberately short timeout
9880/// does not collapse to nothing. A retry that re-sends the whole prompt
9881/// (because the seat kept no context) is the original job again, and keeps the
9882/// original budget.
9883pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9884    if nudged {
9885        (full / 4).max(Duration::from_secs(120)).min(full)
9886    } else {
9887        full
9888    }
9889}
9890
9891/// Run a wave and parse each reply, re-asking the seats whose reply was
9892/// unusable.
9893///
9894/// The re-ask is a nudge rather than the whole prompt again when the seat still
9895/// holds its conversation, which is the difference between a cheap retry and
9896/// paying for the entire candidate set twice.
9897///
9898/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9899/// successor in `roster` is handed to it instead of being re-asked: the
9900/// handover is the retry. A seat with no successor left (a single-agent
9901/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9902/// the asks to one seat in one node number at most
9903/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9904/// is asked once (a dropped stream is nudged first), and only the last agent
9905/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9906/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9907/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9908/// other failure stops the chain when the previous agent failed the same way
9909/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9910/// [`has_context`] is false and the job's own full prompt and full budget are
9911/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9912/// intermediate ones are not losses) and is returned as a failure like any
9913/// other absent seat — the caller decides whether the panel still has a
9914/// quorum. An empty `roster` disables handover: failures are nudged as they
9915/// always were, and a quota is simply lost. A reply that fails to parse or
9916/// validate is the prompt's doing and is only ever nudged, never handed over.
9917///
9918/// The returned [`SeatState`] names the agent that answered (or tried last).
9919#[allow(clippy::too_many_arguments)]
9920async fn ask_json_wave<T>(
9921    jobs: Vec<SeatJob>,
9922    sem: Arc<Semaphore>,
9923    retries: usize,
9924    roster: &[AgentSpec],
9925    ctx: &WaveCtx<'_>,
9926    losses: &mut Vec<QuotaLoss>,
9927    state: &mut RunState,
9928    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9929) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9930where
9931    T: serde::de::DeserializeOwned + Send + 'static,
9932{
9933    ask_wave_with(
9934        jobs,
9935        sem,
9936        retries,
9937        roster,
9938        ctx,
9939        losses,
9940        state,
9941        &|text: &str| {
9942            let v = verdict::extract_json::<T>(text)?;
9943            validate(&v)?;
9944            Ok(v)
9945        },
9946    )
9947    .await
9948}
9949
9950/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9951/// a node whose answer is prose (deliberation) shares the same handover,
9952/// failure classification, quota bookkeeping and bounds instead of a copy.
9953///
9954/// A seat handed to another roster agent is sent the job's `handover` prompt
9955/// (when it has one) rather than `prompt`: the new agent has no session, so a
9956/// resume-style prompt would be incomplete. That holds for the handover ask
9957/// and for every nudge to that agent whose `has_context` is false.
9958#[allow(clippy::too_many_arguments)]
9959async fn ask_wave_with<T>(
9960    jobs: Vec<SeatJob>,
9961    sem: Arc<Semaphore>,
9962    retries: usize,
9963    roster: &[AgentSpec],
9964    ctx: &WaveCtx<'_>,
9965    losses: &mut Vec<QuotaLoss>,
9966    state: &mut RunState,
9967    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9968) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9969where
9970    T: Send + 'static,
9971{
9972    let n = jobs.len();
9973    let originals: Vec<SeatJob> = jobs;
9974    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9975    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9976    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9977    // answer, N once it has gone through N nudges. Read back once this
9978    // returns, so a caller building a history record (`ReviewRecord`) can
9979    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9980    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9981    // field's own doc.
9982    let mut nudges: Vec<usize> = vec![0; n];
9983    // The agent now occupying each seat, the ids it has already been through,
9984    // where in the roster the walk began, the class of the last failure, and
9985    // the stem word of a handover not yet asked (full prompt, full budget).
9986    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9987    let mut tried: Vec<BTreeSet<String>> = specs
9988        .iter()
9989        .map(|s| BTreeSet::from([s.id.clone()]))
9990        .collect();
9991    let starts: Vec<usize> = specs
9992        .iter()
9993        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9994        .collect();
9995    let carry = ctx.carry_seats && !roster.is_empty();
9996    // Carried across rounds: ids that failed the seat earlier, and how the
9997    // last failure went (so a repeat of it is not handed over again).
9998    let carried: Vec<BTreeSet<String>> = originals
9999        .iter()
10000        .map(|j| {
10001            state
10002                .seat_history
10003                .get(&j.seat.key)
10004                .filter(|_| carry)
10005                .map(|h| h.failed.clone())
10006                .unwrap_or_default()
10007        })
10008        .collect();
10009    let mut prev: Vec<Option<FailClass>> = originals
10010        .iter()
10011        .map(|j| {
10012            state
10013                .seat_history
10014                .get(&j.seat.key)
10015                .filter(|_| carry)
10016                .and_then(|h| h.last_fail.clone())
10017        })
10018        .collect();
10019    let next_agent =
10020        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10021            let others: BTreeSet<String> = specs
10022                .iter()
10023                .enumerate()
10024                .filter(|(j, _)| *j != i)
10025                .map(|(_, s)| s.id.clone())
10026                .collect();
10027            pick_successor(
10028                roster,
10029                starts[i],
10030                tried,
10031                carry.then(|| &carried[i]),
10032                &others,
10033            )
10034            .cloned()
10035        };
10036    let mut fresh: Vec<Option<String>> = vec![None; n];
10037    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10038    let mut pending: Vec<usize> = (0..n).collect();
10039
10040    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10041    // asks: an agent with a successor is asked once and handed over, and only
10042    // a seat with no successor spends `retries` nudges on the same agent. This
10043    // only guarantees the loop's own termination whatever those say.
10044    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10045    for round in 0..max_rounds {
10046        if pending.is_empty() {
10047            break;
10048        }
10049        let mut batch = Vec::with_capacity(pending.len());
10050        let mut renudged: Vec<&str> = Vec::new();
10051        for &i in &pending {
10052            let src = &originals[i];
10053            // A seat now held by another agent than the job named has no
10054            // session of its own: it gets the full-context prompt whenever
10055            // it is asked in full (the handover ask, a nudge it cannot
10056            // resume).
10057            let full: &str = match &src.handover {
10058                Some(h) if specs[i].id != src.spec.id => h,
10059                _ => &src.prompt,
10060            };
10061            // The prompt and the budget are one decision: a nudge restates
10062            // finished work, a re-sent prompt redoes it.
10063            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10064                (
10065                    full.to_owned(),
10066                    src.timeout,
10067                    format!("{}-{word}-{}", src.stem, specs[i].id),
10068                )
10069            } else if nudges[i] == 0 {
10070                (full.to_owned(), src.timeout, src.stem.clone())
10071            } else {
10072                renudged.push(src.seat.key.as_str());
10073                let why = done[i]
10074                    .as_ref()
10075                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10076                    .unwrap_or_else(|| "no parsable answer".to_owned());
10077                let nudge = prompt::nudge(&why);
10078                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10079                let prompt = if nudged {
10080                    nudge
10081                } else {
10082                    format!("{full}\n\n---\n\n{nudge}")
10083                };
10084                (
10085                    prompt,
10086                    retry_budget(src.timeout, nudged),
10087                    format!("{}-retry{}", src.stem, nudges[i]),
10088                )
10089            };
10090            batch.push(SeatJob {
10091                spec: specs[i].clone(),
10092                seat: seats[i].clone(),
10093                cwd: src.cwd.clone(),
10094                prompt,
10095                timeout,
10096                allow_write: src.allow_write,
10097                sessions: src.sessions,
10098                artifacts: src.artifacts.clone(),
10099                stem,
10100                handover: None,
10101            });
10102        }
10103
10104        if !renudged.is_empty() {
10105            state.event(
10106                ctx.node,
10107                format!("retry {round}: re-asking {}", renudged.join(", ")),
10108            );
10109        }
10110        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10111        let mut still = Vec::new();
10112        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10113            seats[i] = seat;
10114            // No agent was asked: nothing to hand over, nudge, remember or
10115            // charge. The caller sees `resource_block` and stops the node.
10116            if let AgentOutcome::ResourceBlocked(why) = &out {
10117                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10118                continue;
10119            }
10120            let class = FailClass::of(&out);
10121            // A dropped stream is nudged first (the conversation is still
10122            // there to pick up); only a seat whose nudges are spent hands over.
10123            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10124                && nudges[i] < retries
10125                && !roster.is_empty();
10126            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10127                let next = should_hand_over(prev[i].as_ref(), &cur)
10128                    .then(|| next_agent(i, &tried[i], &specs))
10129                    .flatten();
10130                if carry {
10131                    let h = state
10132                        .seat_history
10133                        .entry(originals[i].seat.key.clone())
10134                        .or_default();
10135                    h.failed.insert(specs[i].id.clone());
10136                    h.last_fail = Some(cur.clone());
10137                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10138                        h.last_ok = None;
10139                    }
10140                    // Saved before the next agent is asked, so a restart in
10141                    // between does not forget who failed.
10142                    if let Err(e) = state.save() {
10143                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10144                    }
10145                }
10146                if let Some(next) = next {
10147                    record_handover(
10148                        state,
10149                        ctx.node,
10150                        &originals[i].seat.key,
10151                        &specs[i].id,
10152                        &next.id,
10153                        &cur,
10154                        &fail_reason(&out),
10155                    );
10156                    tried[i].insert(next.id.clone());
10157                    prev[i] = Some(cur.clone());
10158                    seats[i] =
10159                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10160                    specs[i] = next;
10161                    fresh[i] = Some(cur.stem_word().to_owned());
10162                    nudges[i] = 0;
10163                    done[i] = Some(Err(anyhow::anyhow!(
10164                        "handed over after: {}",
10165                        fail_reason(&out)
10166                    )));
10167                    still.push(i);
10168                    continue;
10169                }
10170            }
10171            if carry
10172                && !nudge_first
10173                && let Some(cur) = class.clone()
10174            {
10175                // The chain ended here (no successor, or a repeated failure).
10176                let h = state
10177                    .seat_history
10178                    .entry(originals[i].seat.key.clone())
10179                    .or_default();
10180                h.failed.insert(specs[i].id.clone());
10181                h.last_fail = Some(cur);
10182            }
10183            let parsed = match out {
10184                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10185                AgentOutcome::Quota(o) => {
10186                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10187                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10188                }
10189                // Not a parseable answer: the nudge loop re-asks it, which is
10190                // exactly what a dropped stream needs. Just don't hand its raw
10191                // error JSON to `extract_json`.
10192                AgentOutcome::Dropped(o) => {
10193                    let why = o
10194                        .dropped
10195                        .as_ref()
10196                        .map(|d| d.why.as_str())
10197                        .unwrap_or("the CLI ended the stream without delivering its answer");
10198                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10199                }
10200                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10201                    Err(anyhow::anyhow!(e))
10202                }
10203            };
10204            let quota = class == Some(FailClass::Quota);
10205            let failed = parsed.is_err();
10206            done[i] = Some(parsed);
10207            if carry && !failed {
10208                let h = state
10209                    .seat_history
10210                    .entry(originals[i].seat.key.clone())
10211                    .or_default();
10212                h.failed.remove(&specs[i].id);
10213                h.last_ok = Some(specs[i].id.clone());
10214                h.last_fail = None;
10215            }
10216            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10217            // fail the same way; and never re-ask a seat that already parsed.
10218            // A failed agent that still has a successor is not re-asked
10219            // either: the handover was its remedy and has just been refused
10220            // (the chain stops on a repeated failure class). A seat with no
10221            // successor left (a single-agent roster, the roster's tail, or an
10222            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10223            let agent_failure = class.is_some()
10224                && !nudge_first
10225                && !roster.is_empty()
10226                && next_agent(i, &tried[i], &specs).is_some();
10227            if failed && !quota && !agent_failure && nudges[i] < retries {
10228                nudges[i] += 1;
10229                still.push(i);
10230            }
10231        }
10232        pending = still;
10233    }
10234
10235    // One loss per seat whose chain ended on a quota: the intermediate ones
10236    // were absorbed by a handover and are not losses.
10237    for (i, q) in last_quota.into_iter().enumerate() {
10238        if let Some(reset) = q {
10239            losses.push(QuotaLoss {
10240                seat: originals[i].seat.key.clone(),
10241                node: ctx.node.to_owned(),
10242                at: Timestamp::now(),
10243                reset,
10244            });
10245        }
10246    }
10247
10248    seats
10249        .into_iter()
10250        .zip(done)
10251        .zip(nudges)
10252        .map(|((seat, res), attempts)| {
10253            (
10254                seat,
10255                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10256                attempts,
10257            )
10258        })
10259        .collect()
10260}
10261
10262/// Did the wave end without asking any agent (see [`block_wave`])?
10263fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10264    results
10265        .iter()
10266        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10267}
10268
10269/// Settle a wave that never spawned anything: every job answers
10270/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10271/// else about the seat (session, history, quota) is touched.
10272fn block_wave(
10273    jobs: Vec<SeatJob>,
10274    state: &mut RunState,
10275    node: &str,
10276    attempt: usize,
10277    why: &str,
10278) -> Vec<(usize, SeatState, AgentOutcome)> {
10279    state.resource_wait = None;
10280    state
10281        .active
10282        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10283    if let Err(e) = state.save() {
10284        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10285    }
10286    jobs.into_iter()
10287        .enumerate()
10288        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10289        .collect()
10290}
10291
10292/// Acquire the shared build cache's lease, waiting out contention within
10293/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10294/// unbounded wait is never acceptable).
10295///
10296/// A first, non-blocking check happens before ever waiting; if it finds the
10297/// lease busy, that fact is logged as a `verify` event *and* flushed with
10298/// [`RunState::save`] immediately — not only once the wait finally succeeds
10299/// or gives up — so a `magi show` run by a different process while this one
10300/// is still waiting reads a `run.json` that says so, rather than whatever it
10301/// looked like before the wait started. The same applies to the terminal
10302/// failure: logged and saved before this returns `Err`, so a caller that
10303/// could not get the lease at all still leaves a legible record of why.
10304async fn acquire_cache_lease(
10305    state: &mut RunState,
10306    cache_dir: &Path,
10307    owner: &crate::cache::Owner,
10308    budget: Duration,
10309    node: &str,
10310    context: &str,
10311    cancel: &(dyn Fn() -> bool + Sync),
10312) -> Result<Option<crate::cache::Guard>> {
10313    let home = crate::run::home();
10314    let started = Instant::now();
10315    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10316        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10317        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10318        Err(e) => {
10319            state.event(
10320                "verify",
10321                format!("{context}: could not check the shared build cache: {e:#}"),
10322            );
10323            if let Err(e2) = state.save() {
10324                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10325            }
10326            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10327        }
10328    };
10329    state.event(
10330        "verify",
10331        format!(
10332            "{context}: waiting for the shared build cache at {} ({})",
10333            cache_dir.display(),
10334            busy.describe()
10335        ),
10336    );
10337    // Seats of this node are waiting, not running: say so, so the countdown a
10338    // reader sees only ever measures an agent that has been spawned.
10339    state.resource_wait = Some(crate::run::ResourceWait {
10340        node: context.to_owned(),
10341        kind: "shared build cache".to_owned(),
10342        holder: busy.describe(),
10343        since: Timestamp::now(),
10344        limit_secs: budget.as_secs(),
10345    });
10346    if let Err(e) = state.save() {
10347        tracing::warn!("could not persist a cache wait: {e:#}");
10348    }
10349    let remaining = budget.saturating_sub(started.elapsed());
10350    let waited = crate::cache::wait_for_unless(
10351        &home,
10352        cache_dir,
10353        owner,
10354        remaining,
10355        Duration::from_secs(5),
10356        cancel,
10357    )
10358    .await;
10359    state.resource_wait = None;
10360    let cause = match &waited {
10361        Ok(Some(_)) => None,
10362        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10363        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10364            Some(crate::run::ResourceBlockCause::Timeout {
10365                waited_secs: started.elapsed().as_secs(),
10366                limit_secs: budget.as_secs(),
10367            })
10368        }
10369        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10370    };
10371    if let Some(cause) = cause {
10372        let detail = match &waited {
10373            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10374            Err(error) => format!("{error:#}"),
10375        };
10376        state.resource_block = Some(crate::run::ResourceBlock {
10377            cause,
10378            node: node.to_owned(),
10379            kind: "shared build cache".into(),
10380            detail,
10381            at: Timestamp::now(),
10382        });
10383        state.event(
10384            "resource",
10385            state.resource_block.as_ref().unwrap().description(),
10386        );
10387    }
10388    if let Err(error) = state.save() {
10389        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10390    }
10391    waited
10392}
10393
10394/// The lease state could not be read at all (unreadable lease file, a
10395/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10396/// known to be contended, so a wave degrades to running without a shared cache
10397/// instead of stopping on a hold that [`crate::triage`] could never release.
10398#[derive(Debug)]
10399struct CacheCheckError(String);
10400
10401impl std::fmt::Display for CacheCheckError {
10402    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10403        f.write_str(&self.0)
10404    }
10405}
10406
10407impl std::error::Error for CacheCheckError {}
10408
10409/// Keep a specific acquisition outcome when the caller adds command evidence.
10410fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10411    state
10412        .resource_block
10413        .get_or_insert_with(|| crate::run::ResourceBlock {
10414            cause: crate::run::ResourceBlockCause::Other,
10415            node: node.to_owned(),
10416            kind: "shared build cache".into(),
10417            detail: format!("{error:#}"),
10418            at: Timestamp::now(),
10419        });
10420    if let Err(error) = state.save() {
10421        tracing::warn!("could not persist cache resource block: {error:#}");
10422    }
10423}
10424
10425/// Run `body` — a verify command batch — while holding the shared build
10426/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10427/// never interleave with another borrower's build against the same
10428/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10429/// timeout, or a human's own `magi review`. See the `cache` module doc for
10430/// why this matters more than Cargo's own per-target locking covers — two
10431/// *different* worktrees building the same package name/version into one
10432/// cache directory is a staleness bug, not a lock contention one.
10433///
10434/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10435/// agent waves do. Repository scheduling also has its own bounded,
10436/// cancellable resource-wait policy. Contention must not consume the
10437/// verification budget: `body` receives the full `budget` after both leases
10438/// are acquired and the cache is fresh. Without a shared cache, only
10439/// repository coordination is acquired before handing over the full budget.
10440///
10441/// A lease that cannot be acquired within the resource wait is reported as a single
10442/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10443/// verification — the same shape a spawn failure already takes in
10444/// [`run_commands`], so a caller need not special-case it.
10445#[allow(clippy::too_many_arguments)]
10446async fn with_cache_lease<'s, F, Fut>(
10447    state: &'s mut RunState,
10448    cache_dir: Option<&Path>,
10449    node: &str,
10450    seat: &str,
10451    worktree: &Path,
10452    head: &str,
10453    budget: Duration,
10454    context: &str,
10455    wait: &crate::repo_lock::WaitPolicy,
10456    body: F,
10457) -> Result<(Vec<CommandOutcome>, bool)>
10458where
10459    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10460    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10461{
10462    // Match editing waves: cache first, repository second. In particular,
10463    // pre_gate must never hold the repository while waiting for a wave's cache.
10464    let started = Instant::now();
10465    let home = crate::run::home();
10466    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10467    let guard = if let Some(cache_dir) = cache_dir {
10468        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10469        match acquire_cache_lease(
10470            state,
10471            cache_dir,
10472            &owner,
10473            resource_limit,
10474            node,
10475            context,
10476            &|| wait.cancelled(),
10477        )
10478        .await
10479        .and_then(|g| {
10480            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10481        }) {
10482            Ok(g) => Some(g),
10483            Err(e) => {
10484                record_cache_error(state, node, &e);
10485                return Ok((
10486                    vec![CommandOutcome {
10487                        command: "(waiting for the shared build cache)".into(),
10488                        code: None,
10489                        output_tail: e.to_string(),
10490                        duration_ms: started.elapsed().as_millis() as u64,
10491                        resource_blocked: true,
10492                    }],
10493                    false,
10494                ));
10495            }
10496        }
10497    } else {
10498        None
10499    };
10500    let _coordination = coordinate_operation(state, worktree, wait).await?;
10501    if let Some(cache_dir) = cache_dir {
10502        let identity = crate::cache::Identity::new(worktree, head);
10503        if let Err(e) = crate::cache::ensure_fresh(
10504            &home,
10505            cache_dir,
10506            &identity,
10507            &state.config.verify.cargo_manifests(),
10508        ) {
10509            record_cache_error(state, node, &e);
10510            state.event(
10511                "verify",
10512                format!(
10513                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10514                    worktree.display(),
10515                    short(head)
10516                ),
10517            );
10518            if let Some(guard) = guard {
10519                guard.release();
10520            }
10521            return Ok((
10522                vec![CommandOutcome {
10523                    command: "(confirming the shared build cache is fresh)".into(),
10524                    code: None,
10525                    output_tail: e.to_string(),
10526                    duration_ms: started.elapsed().as_millis() as u64,
10527                    resource_blocked: true,
10528                }],
10529                false,
10530            ));
10531        }
10532    }
10533    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10534    if !timed_out_pids.is_empty() {
10535        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10536    }
10537    // Drop repository coordination before releasing the cache, reversing the
10538    // acquisition order. A cancelled/error future also drops both guards.
10539    drop(_coordination);
10540    if let Some(guard) = guard {
10541        guard.release();
10542    }
10543    Ok((outcomes, retried))
10544}
10545
10546/// Poll `pids` — commands [`run_commands`] reports as still running when its
10547/// own timeout elapsed — until every one is confirmed gone, or
10548/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10549///
10550/// Real confirmation where confirmation is possible, not a substitute for
10551/// full process-tree observation: a grandchild the timed-out process spawned
10552/// and that survives independently of it is invisible to a pid check the
10553/// same way it always was, and continuing to observe and collect *that*
10554/// stays a different piece of work with its own owner. This only narrows a
10555/// fixed blind wait into an actual check of the pids this process does know
10556/// about.
10557async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10558    wait_for_pids_with(
10559        pids,
10560        crate::proc::pid_alive,
10561        LEASE_RELEASE_POLL,
10562        LEASE_RELEASE_MAX_WAIT,
10563    )
10564    .await;
10565}
10566
10567/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10568/// interval and ceiling supplied by the caller, so the polling *logic* -
10569/// returns as soon as every pid reports dead, gives up at the ceiling
10570/// otherwise - is testable on millisecond durations without asking the real
10571/// OS about a pid at all.
10572async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10573    pids: &[u32],
10574    alive: F,
10575    poll: Duration,
10576    max_wait: Duration,
10577) {
10578    let deadline = Instant::now() + max_wait;
10579    loop {
10580        if pids.iter().all(|&pid| !alive(pid)) {
10581            return;
10582        }
10583        if Instant::now() >= deadline {
10584            return;
10585        }
10586        tokio::time::sleep(poll).await;
10587    }
10588}
10589
10590/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10591/// admission that it could not even get a verify command to run, as opposed
10592/// to evidence the command actually produced? A caller that would otherwise
10593/// read a resource-blocked outcome as a red command must check this first:
10594/// see [`Runner::gate`], which retries rather than records `Blocked` when
10595/// this is true.
10596fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10597    outcomes.iter().any(|o| o.resource_blocked)
10598}
10599
10600/// What [`Runner::gate_fix_round`] decided.
10601pub(crate) enum GateFix {
10602    /// The tree changed and `verify.e2e` is still green: run the gate again.
10603    Retry,
10604    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10605    /// last failure stands and the run ends blocked.
10606    Stop,
10607    /// `verify.e2e` could not run after the fix (magi's own contention):
10608    /// decide nothing now, a later reentry retries.
10609    Defer,
10610}
10611
10612/// Is every red command in `outcomes` an ordinary failure the code could
10613/// explain: it ran, exited non-zero, and said something?
10614///
10615/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10616/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10617/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10618/// what the command is stays the gate's business.
10619fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10620    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10621    red.peek().is_some()
10622        && red.all(|o| {
10623            !o.resource_blocked
10624                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10625                && !o.output_tail.trim().is_empty()
10626        })
10627}
10628
10629/// Describe one verify command's outcome for the event log, distinguishing a
10630/// build/link failure — the toolchain never produced a binary to run — from
10631/// an actual test failure, since only the latter is a verdict on the patch.
10632fn e2e_outcome_label(o: &CommandOutcome) -> String {
10633    if o.ok() {
10634        return "pass".to_owned();
10635    }
10636    let reason = if o.build_failed() {
10637        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10638    } else {
10639        format!("FAIL ({:?})", o.code)
10640    };
10641    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10642}
10643
10644/// Run `verify.e2e`, retrying once if the first attempt could not build or
10645/// link — a build/link failure is frequently a race against a shared
10646/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10647/// `verify` event per command, tagged with `context` (normally `"round N"`)
10648/// so the two call sites that need this — the ordinary per-round leg in
10649/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10650/// it will ever call a round green — read identically in the event log.
10651async fn run_e2e_with_retry(
10652    state: &mut RunState,
10653    shell: &[String],
10654    commands: &[String],
10655    worktree: &Path,
10656    timeout: Duration,
10657    context: &str,
10658) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10659    let (mut e2e, mut timed_out_pids) = run_commands(
10660        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10661    )
10662    .await;
10663    for o in &e2e {
10664        state.event(
10665            "verify",
10666            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10667        );
10668    }
10669    // A build/link failure is not a verdict on the patch — it is frequently a
10670    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10671    // one retry before letting a red like that decide the round.
10672    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10673    if verify_retried {
10674        state.event(
10675            "verify",
10676            format!(
10677                "{context}: verify could not build/link, not a test result — retrying once \
10678                 before concluding"
10679            ),
10680        );
10681        let retried = run_commands(
10682            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10683        )
10684        .await;
10685        e2e = retried.0;
10686        // Both attempts' timeouts matter, not just the last one: the first
10687        // attempt's descendants may still be alive alongside the retry's.
10688        timed_out_pids.extend(retried.1);
10689        for o in &e2e {
10690            state.event(
10691                "verify",
10692                format!(
10693                    "{context}: retry `{}` -> {}",
10694                    o.command,
10695                    e2e_outcome_label(o)
10696                ),
10697            );
10698        }
10699    }
10700    (e2e, verify_retried, timed_out_pids)
10701}
10702
10703/// Run configured shell commands in `cwd`, in order. The second element is
10704/// the pid of every command that hit `timeout` and was still running when
10705/// this stopped waiting on it (best-effort: `None` when the platform did not
10706/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10707/// that releases a shared resource afterward needs to know.
10708///
10709/// Records `task` into [`RunState::active`] at every command boundary
10710/// (`RunState::task_command`) and clears it once the whole list has run
10711/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10712/// for minutes with no seat and no output of its own to show for it (see
10713/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10714/// apart from "not yet run" without this), and this is the only place that
10715/// knows which command is running right now and how many are left. Three
10716/// saves per command — start, not per second — matching the same "only at a
10717/// boundary" rule [`wave`] already follows for seats.
10718#[allow(clippy::too_many_arguments)]
10719async fn run_commands(
10720    state: &mut RunState,
10721    node: &str,
10722    task: &str,
10723    attempt: usize,
10724    shell: &[String],
10725    commands: &[String],
10726    cwd: &Path,
10727    timeout: Duration,
10728) -> (Vec<CommandOutcome>, Vec<u32>) {
10729    if commands.is_empty() {
10730        // Nothing to mark as running and nothing to clear — an empty list
10731        // means "not configured", and touching `active` (or the disk) over
10732        // that would be a write for every round of a repo with no
10733        // `verify.e2e` / `verify.gate` commands at all.
10734        return (Vec::new(), Vec::new());
10735    }
10736    let mut out = Vec::new();
10737    let mut timed_out_pids = Vec::new();
10738    let total = commands.len();
10739    for (idx, command) in commands.iter().enumerate() {
10740        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10741        if let Err(e) = state.save() {
10742            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10743        }
10744        let started = Instant::now();
10745        let mut cmd = tokio::process::Command::new(&shell[0]);
10746        cmd.quiet();
10747        cmd.args(&shell[1..])
10748            .arg(command)
10749            .current_dir(cwd)
10750            .stdin(std::process::Stdio::null())
10751            .stdout(std::process::Stdio::piped())
10752            .stderr(std::process::Stdio::piped())
10753            .kill_on_drop(true);
10754        let mut outputs = Vec::new();
10755        let protected = (|| -> Result<()> {
10756            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10757                outputs.push(guard);
10758            }
10759            if let Some(cache) = state.config.cache_dir() {
10760                outputs.push(crate::retention::OutputLease::cache(
10761                    &crate::run::home(),
10762                    &cache,
10763                    false,
10764                )?);
10765            }
10766            Ok(())
10767        })();
10768        if let Err(e) = protected {
10769            out.push(CommandOutcome {
10770                command: command.clone(),
10771                code: None,
10772                output_tail: format!("output ownership unavailable: {e:#}"),
10773                duration_ms: 0,
10774                resource_blocked: true,
10775            });
10776            continue;
10777        }
10778        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10779        let (code, body) = match spawned {
10780            Ok(mut running) => {
10781                let pid = running.child().id();
10782                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10783                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10784                let mut paths = vec![cwd.to_owned()];
10785                if let Some(cache) = state.config.cache_dir() {
10786                    paths.push(cache);
10787                }
10788                let mut pressure = None;
10789                let status = tokio::select! {
10790                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10791                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10792                        pressure = Some(reason); None
10793                    }
10794                };
10795                let (code, failure) = match status {
10796                    Some(Ok(Ok(status))) => (
10797                        status.code(),
10798                        status
10799                            .code()
10800                            .is_none()
10801                            .then(|| format!("command terminated: {status}")),
10802                    ),
10803                    Some(Ok(Err(error))) => {
10804                        (None, Some(format!("failed waiting for command: {error}")))
10805                    }
10806                    Some(Err(_)) => {
10807                        // Leased cleanup confirms process-tree death itself.
10808                        // Unleased commands still need the caller's PID check
10809                        // after kill-on-drop, before releasing cache ownership.
10810                        if !running.coordinated()
10811                            && let Some(pid) = pid
10812                        {
10813                            timed_out_pids.push(pid);
10814                        }
10815                        (
10816                            None,
10817                            Some(format!("timed out after {}s", timeout.as_secs())),
10818                        )
10819                    }
10820                    None => (None, None),
10821                };
10822                running.finish().await;
10823                if let Some(reason) = pressure {
10824                    state.resource_block = Some(crate::run::ResourceBlock {
10825                        cause: crate::run::ResourceBlockCause::Other,
10826                        node: node.to_owned(),
10827                        kind: "disk pressure".to_owned(),
10828                        detail: reason.clone(),
10829                        at: Timestamp::now(),
10830                    });
10831                    out.push(CommandOutcome {
10832                        command: command.clone(),
10833                        code: None,
10834                        output_tail: reason,
10835                        duration_ms: started.elapsed().as_millis() as u64,
10836                        resource_blocked: true,
10837                    });
10838                    break;
10839                }
10840                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10841                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10842                if let Some(failure) = failure {
10843                    body.push('\n');
10844                    body.push_str(&failure);
10845                }
10846                (code, body)
10847            }
10848            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10849        };
10850        out.push(CommandOutcome {
10851            command: command.clone(),
10852            code,
10853            output_tail: tail(&body, OUTPUT_TAIL),
10854            duration_ms: started.elapsed().as_millis() as u64,
10855            resource_blocked: false,
10856        });
10857    }
10858    state.task_finished(task);
10859    if let Err(e) = state.save() {
10860        tracing::warn!("could not persist the end of {task}: {e:#}");
10861    }
10862    (out, timed_out_pids)
10863}
10864
10865/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10866/// section (`report::run`) and in the `merge` event this node records — for
10867/// the operator to run by hand.
10868///
10869/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10870/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10871/// merge commits", or "require linear history") rejects the push a `--no-ff`
10872/// merge would produce, which is exactly the guidance this function replaces.
10873/// `message`'s first line becomes the squash commit's subject, matching the
10874/// note `report::run` prints alongside this command — see that function for
10875/// why an explicit subject is not optional there.
10876fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10877    let repo = repo.display();
10878    match style {
10879        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10880        MergeStyle::Squash => {
10881            // The subject sits inside double quotes, and a title an agent
10882            // wrote may carry the characters that break out of them.
10883            let subject = message
10884                .lines()
10885                .next()
10886                .unwrap_or(branch)
10887                .replace(['\\', '"', '$', '`'], "");
10888            format!(
10889                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10890            )
10891        }
10892        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10893    }
10894}
10895
10896/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10897/// under the hood, rejects a `title` over 256 characters and the whole
10898/// command fails — no PR at all, for a run whose body was otherwise fine
10899/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10900/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10901/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10902/// this bound. It is a margin, not a guarantee — a title packed
10903/// with multi-unit characters could still in principle land close to the
10904/// edge, but a real task title's occasional emoji or accented letter fits
10905/// comfortably inside it.
10906const PR_TITLE_MAX: usize = 240;
10907
10908/// A pull request title from the opening line of `text`, or `None` when that
10909/// line is not English (GitHub text is) or has no letters.
10910///
10911/// A line within `max` is kept as is. A longer one is cut at the end of its
10912/// first sentence when that falls inside `max`, else at a word boundary with a
10913/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10914/// make every merely-truncated title look non-English). The language check
10915/// runs on the kept text before any mark is added, so only what GitHub will
10916/// show is judged: an English opening followed by non-ASCII far past the cut
10917/// still passes.
10918fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10919    let line = queue::first_line(text)?;
10920    let chars: Vec<char> = line.chars().collect();
10921    let (kept, mark) = if chars.len() <= max {
10922        (line.to_owned(), "")
10923    } else if let Some(end) = sentence_end(&chars, max) {
10924        (chars[..end].iter().collect::<String>(), "")
10925    } else {
10926        let room = max.saturating_sub(3);
10927        // Cut at the last space inside the room; when the char just past the
10928        // room is a space the room already ends on a word.
10929        let cut = if chars[room].is_whitespace() {
10930            room
10931        } else {
10932            chars[..room]
10933                .iter()
10934                .rposition(|c| c.is_whitespace())
10935                .unwrap_or(room)
10936        };
10937        let head: String = chars[..cut].iter().collect();
10938        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10939        (head.to_owned(), "...")
10940    };
10941    if kept.is_empty() {
10942        return None;
10943    }
10944    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10945        return None;
10946    }
10947    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10948        return None;
10949    }
10950    Some(format!("{kept}{mark}"))
10951}
10952
10953/// The char length of the first sentence of `chars` when it ends within `max`
10954/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10955/// abbreviations so `e.g. foo` does not end a title early.
10956fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10957    const MIN: usize = 20;
10958    for i in MIN..max.min(chars.len()) {
10959        if !matches!(chars[i], '.' | '!' | '?') {
10960            continue;
10961        }
10962        let Some(&next) = chars.get(i + 1) else {
10963            continue;
10964        };
10965        if !next.is_whitespace() {
10966            continue;
10967        }
10968        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10969        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10970            continue;
10971        }
10972        let word: String = chars[..i]
10973            .iter()
10974            .rev()
10975            .take_while(|c| !c.is_whitespace())
10976            .collect::<Vec<_>>()
10977            .into_iter()
10978            .rev()
10979            .collect();
10980        let word = word.to_ascii_lowercase();
10981        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10982            continue;
10983        }
10984        let end = chars[..i]
10985            .iter()
10986            .rposition(|c| !c.is_whitespace())
10987            .map_or(i, |p| p + 1);
10988        return Some(end);
10989    }
10990    None
10991}
10992
10993/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10994/// change: a title and a body describing what was *implemented*, not the task
10995/// that asked for it. A task reads as a request; a reader of the merged
10996/// history wants the change.
10997struct PrMessage {
10998    title: String,
10999    body: String,
11000}
11001
11002impl PrMessage {
11003    /// Title, blank line, body. The first line is the squash/merge commit
11004    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11005    /// has to stay one sensible line.
11006    fn commit_message(&self) -> String {
11007        format!("{}\n\n{}", self.title, self.body)
11008    }
11009}
11010
11011/// The text after a leading `TITLE:` (any case) on `line`.
11012fn title_marker(line: &str) -> Option<&str> {
11013    let line = line.trim();
11014    let head = line.get(..6)?;
11015    head.eq_ignore_ascii_case("title:")
11016        .then(|| line[6..].trim())
11017}
11018
11019/// The implementer's own one-line title: the `TITLE:` line the implement
11020/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11021/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11022/// source, and a title that says as much is refused here too.
11023fn summary_title(summary: &str) -> Option<String> {
11024    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11025    let raw = title_marker(first)?;
11026    if raw.is_empty() {
11027        return None;
11028    }
11029    let title = queue::title_from(raw, PR_TITLE_MAX);
11030    let lower = title.to_ascii_lowercase();
11031    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11032        return None;
11033    }
11034    Some(title)
11035}
11036
11037/// How `open_review`'s instruction begins; see [`landing_title`].
11038const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11039
11040/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11041pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11042
11043/// Does this run review work that already existed, rather than implement a
11044/// task? Runs recorded before `reviewed_commits` existed carry only the
11045/// candidate marker.
11046pub(crate) fn is_review_run(state: &RunState) -> bool {
11047    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11048}
11049
11050/// The title of a review-only run: the subject of the oldest commit under
11051/// review. Later commits are usually fixups, and `instruction` is the review
11052/// prompt, which says nothing about the change. GitHub text is English, so a
11053/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11054fn review_title(state: &RunState) -> Option<String> {
11055    english_subject(
11056        state.reviewed_commits.as_ref()?.first()?,
11057        state.config.graph.github_english(),
11058    )
11059}
11060
11061/// `raw` as a pull request title, or `None` when it is blank, not English
11062/// (GitHub text is), or one of magi's own candidate commit subjects.
11063fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11064    let raw = raw.trim();
11065    if raw.is_empty() {
11066        return None;
11067    }
11068    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11069        return None;
11070    }
11071    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11072        return None;
11073    }
11074    let title = queue::title_from(raw, PR_TITLE_MAX);
11075    let lower = title.to_ascii_lowercase();
11076    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11077        return None;
11078    }
11079    Some(title)
11080}
11081
11082/// What a review-only run's branch says about itself, read at the moment the
11083/// pull request is opened.
11084#[derive(Debug, Clone, PartialEq, Eq)]
11085struct BranchFacts {
11086    /// `(subject, body)` of each commit, oldest first.
11087    commits: Vec<(String, String)>,
11088    /// Trimmed `git diff --stat`.
11089    stat: String,
11090}
11091
11092/// Longest diff stat shown: this many file lines plus the summary line.
11093const STAT_FILE_LINES: usize = 25;
11094/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11095const COMMITS_MAX_CHARS: usize = 20_000;
11096
11097/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11098/// say or finds nothing, so the caller falls back to what the run recorded.
11099async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11100    let commits = git::commit_log(repo, base, branch).await.ok()?;
11101    if commits.is_empty() {
11102        return None;
11103    }
11104    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11105    let lines: Vec<&str> = stat.lines().collect();
11106    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11107        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11108        let more = format!(" ... {omitted} more file(s)");
11109        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11110        kept.push(&more);
11111        kept.push(lines[lines.len() - 1]);
11112        kept.join("\n")
11113    } else {
11114        lines.join("\n")
11115    };
11116    Some(BranchFacts { commits, stat })
11117}
11118
11119/// Defang what would break the surrounding markdown: a closing `</details>`
11120/// and a code fence.
11121fn markdown_safe(text: &str) -> String {
11122    text.replace("</details>", "&lt;/details&gt;")
11123        .replace("\x60\x60\x60", "~~~")
11124}
11125
11126fn neutral_title(state: &RunState, winner: char) -> String {
11127    format!(
11128        "chore: land candidate {} of run {}",
11129        winner.to_ascii_uppercase(),
11130        state.id
11131    )
11132}
11133
11134/// Reuse the graph's bounded fixer without starting another graph or driver.
11135/// Copy the working state back even on errors so spent rounds remain visible.
11136pub(crate) async fn repair_landing_gate(
11137    state: &mut RunState,
11138    winner: &Candidate,
11139    outcomes: &[CommandOutcome],
11140) -> Result<GateFix> {
11141    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11142        return Ok(GateFix::Stop);
11143    }
11144    let mut runner = Runner {
11145        roles: state.config.resolve_roles()?,
11146        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11147        state: state.clone(),
11148        pause: Pause::new(),
11149        interrupt: Pause::new(),
11150    };
11151    let result = runner.gate_fix_round(winner, outcomes).await;
11152    *state = runner.state;
11153    result
11154}
11155
11156/// Recheck a repair's e2e requirement across failures, resource deferrals and
11157/// process interruptions. A successful result only applies to its exact SHA.
11158pub(crate) async fn check_gate_repair_e2e(
11159    state: &mut RunState,
11160    winner: &Candidate,
11161    wait: &crate::repo_lock::WaitPolicy,
11162) -> Result<GateFix> {
11163    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11164        return Ok(GateFix::Retry);
11165    }
11166    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11167    if state
11168        .gate_repair_e2e
11169        .as_ref()
11170        .is_some_and(|r| r.head == head && r.passed)
11171    {
11172        return Ok(GateFix::Retry);
11173    }
11174    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11175        head: head.clone(),
11176        passed: false,
11177        outcomes: Vec::new(),
11178    });
11179    state.save()?;
11180    let commands = state.config.verify.e2e.clone();
11181    let outcomes = if commands.is_empty() {
11182        Vec::new()
11183    } else {
11184        let shell = state.config.shell();
11185        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11186        let cache_dir = state.config.cache_dir();
11187        let (outcomes, _) = with_cache_lease(
11188            state,
11189            cache_dir.as_deref(),
11190            "e2e",
11191            "e2e",
11192            &winner.worktree,
11193            &head,
11194            timeout,
11195            "gate repair e2e",
11196            wait,
11197            |state, budget| async move {
11198                run_e2e_with_retry(
11199                    state,
11200                    &shell,
11201                    &commands,
11202                    &winner.worktree,
11203                    budget,
11204                    "gate repair e2e",
11205                )
11206                .await
11207            },
11208        )
11209        .await?;
11210        outcomes
11211    };
11212    let deferred = verify_inconclusive(&outcomes);
11213    let passed = outcomes.iter().all(CommandOutcome::ok)
11214        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11215    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11216        head,
11217        passed,
11218        outcomes,
11219    });
11220    state.save()?;
11221    if deferred {
11222        Ok(GateFix::Defer)
11223    } else if !passed {
11224        state.event(
11225            "gate",
11226            "verify.e2e failed after gate repair; nothing published",
11227        );
11228        Ok(GateFix::Stop)
11229    } else {
11230        Ok(GateFix::Retry)
11231    }
11232}
11233
11234/// Run the same configured gate for a landing head and a fresh graph.
11235pub(crate) async fn run_landing_gate(
11236    state: &mut RunState,
11237    winner: &Candidate,
11238    wait: &crate::repo_lock::WaitPolicy,
11239) -> Result<Vec<CommandOutcome>> {
11240    let shell = state.config.shell();
11241    let gate_commands = state.config.verify.gate.clone();
11242    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11243    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11244        Vec::new()
11245    } else {
11246        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11247        let cache_dir = state.config.cache_dir();
11248        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11249        let (outcomes, _) = with_cache_lease(
11250            state,
11251            cache_dir.as_deref(),
11252            "gate",
11253            "gate",
11254            &winner.worktree,
11255            &head,
11256            timeout,
11257            "final gate",
11258            wait,
11259            |state, budget| {
11260                let shell = shell.clone();
11261                let gate_commands = gate_commands.clone();
11262                let winner = winner.clone();
11263                async move {
11264                    Runner::run_pre_gate(state, &winner).await;
11265                    let (outcomes, timed_out_pids) = run_commands(
11266                        state,
11267                        "gate",
11268                        "gate",
11269                        0,
11270                        &shell,
11271                        &gate_commands,
11272                        &winner.worktree,
11273                        budget,
11274                    )
11275                    .await;
11276                    (outcomes, false, timed_out_pids)
11277                }
11278            },
11279        )
11280        .await?;
11281        outcomes
11282    };
11283    if outcomes.is_empty() {
11284        // Nothing configured to check — distinct from every other
11285        // silence in this run's event log, since an empty `gate` alone
11286        // no longer says whether the gate ran at all (see
11287        // `RunState::gate_ran`'s own doc).
11288        state.event("gate", "no gate commands configured; not tested");
11289    }
11290    for o in &outcomes {
11291        state.event(
11292            "gate",
11293            format!(
11294                "`{}` -> {}",
11295                o.command,
11296                if o.ok() {
11297                    "pass".to_owned()
11298                } else {
11299                    format!(
11300                        "FAIL ({:?})\n{}",
11301                        o.code,
11302                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11303                    )
11304                }
11305            ),
11306        );
11307    }
11308    Ok(outcomes)
11309}
11310
11311/// The pull request title to hand to `land::merge_subject`. A review-only run
11312/// opened by an earlier build titled its pull request with the review prompt;
11313/// that title is dropped (empty, so the fallback applies) rather than landed.
11314/// Any other title, including an operator's rename, passes through untouched,
11315/// and so does every title of a run that implements a task.
11316pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11317    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11318        ""
11319    } else {
11320        pr_title
11321    }
11322}
11323
11324/// What the squash subject falls back to when the pull request title is empty
11325/// or candidate-shaped: for a review-only run the derived title, never the
11326/// review prompt held in `instruction`.
11327pub fn landing_subject_source(state: &RunState) -> String {
11328    if is_review_run(state) {
11329        let winner = state.candidates.first().map_or('A', |c| c.label);
11330        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11331    }
11332    state.instruction.clone()
11333}
11334
11335/// `summary` without its `TITLE:` line, which the pull request title already
11336/// carries.
11337fn summary_without_title(summary: &str) -> String {
11338    let mut lines = summary.trim().lines().peekable();
11339    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11340        lines.next();
11341    }
11342    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11343}
11344
11345/// The pull request title and body for the winning candidate.
11346///
11347/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11348/// the task's own opening line via [`queue::title_from`] when there is none.
11349/// `state.instruction` can open with blank lines (`task_text` only rejects a
11350/// body that is blank *entirely*), which `title_from` skips.
11351///
11352/// Publishing uses an independent description generated from the final diff
11353/// and structured evidence. Internal summaries and reports remain local.
11354/// Non-publishing previews may display the original task and candidate summary.
11355/// The footer retains the run and candidate provenance.
11356#[cfg(test)]
11357fn pr_message(state: &RunState, winner: char) -> PrMessage {
11358    pr_message_with(state, winner, None)
11359}
11360
11361/// [`pr_message`] with what the branch of a review-only run says about itself.
11362/// `facts` is ignored for a run that implements a task.
11363#[cfg(test)]
11364fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11365    let raw = pr_message_raw(
11366        state,
11367        winner,
11368        facts,
11369        crate::pr_outcome::FinalHead::Unreported,
11370        ChangeNote::NotAsked,
11371    );
11372    let id = crate::scrub::Identity::current();
11373    PrMessage {
11374        title: crate::scrub::scrub(&raw.title, &id),
11375        body: crate::scrub::scrub(&raw.body, &id),
11376    }
11377}
11378
11379/// What the public description may say about the change itself.
11380#[derive(Debug, Clone, Copy)]
11381enum ChangeNote<'a> {
11382    /// A local preview or non-publishing merge mode: no public generation needed.
11383    NotAsked,
11384    /// A summary written from the final diff.
11385    Summary(&'a crate::run::ChangeSummary),
11386    /// A summary was asked for and could not be produced; the text says why.
11387    Unavailable(&'a str),
11388}
11389
11390fn pr_message_raw(
11391    state: &RunState,
11392    winner: char,
11393    facts: Option<&BranchFacts>,
11394    head: crate::pr_outcome::FinalHead<'_>,
11395    change: ChangeNote<'_>,
11396) -> PrMessage {
11397    if !matches!(change, ChangeNote::NotAsked) {
11398        let (title, mut body) = match change {
11399            ChangeNote::Summary(c) => (
11400                c.title.clone(),
11401                format!("## Summary\n\n{}\n\n", c.body.trim()),
11402            ),
11403            ChangeNote::Unavailable(why) => (
11404                neutral_title(state, winner),
11405                format!(
11406                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11407                ),
11408            ),
11409            ChangeNote::NotAsked => unreachable!(),
11410        };
11411        body.push_str(&public_pr_evidence(state, head));
11412        body.push_str(&format!(
11413            "\n---\nmagi:run/{} magi:candidate-{}\n",
11414            state.id,
11415            winner.to_ascii_lowercase()
11416        ));
11417        return PrMessage { title, body };
11418    }
11419    pr_preview(state, winner, facts, head)
11420}
11421
11422/// Append public facts from the run records after every prose generation,
11423/// including posting-gate rewrites. The agent cannot drop this provenance.
11424fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11425    let mut body = crate::pr_outcome::review_rounds(state);
11426    body.push_str(&crate::pr_outcome::final_verification(state, head));
11427    let open = state.last_round_findings();
11428    if !open.is_empty() {
11429        body.push_str("## Open review findings\n\n");
11430        for f in open {
11431            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11432        }
11433        body.push('\n');
11434    }
11435    if let Some(c) = state.change_summary.as_ref()
11436        && head == crate::pr_outcome::FinalHead::At(&c.head)
11437    {
11438        body.push_str(&format!(
11439            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11440            c.base, c.head
11441        ));
11442    }
11443    body
11444}
11445
11446/// Local-only preview of the operator's task and candidate summary.
11447fn pr_preview(
11448    state: &RunState,
11449    winner: char,
11450    facts: Option<&BranchFacts>,
11451    head: crate::pr_outcome::FinalHead<'_>,
11452) -> PrMessage {
11453    let summary = state
11454        .candidates
11455        .iter()
11456        .find(|c| c.label == winner)
11457        .map(|c| c.summary.as_str())
11458        .unwrap_or_default();
11459    // The fallback is the operator's own words and may not be English; GitHub
11460    // text always is, so a non-English task gets a neutral title instead.
11461    let review = is_review_run(state);
11462    let title = if review {
11463        facts
11464            .and_then(|f| {
11465                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11466            })
11467            .or_else(|| review_title(state))
11468            .or_else(|| {
11469                state
11470                    .candidates
11471                    .iter()
11472                    .find(|c| c.label == winner)
11473                    .filter(|c| !c.branch.starts_with("magi/"))
11474                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11475            })
11476            .unwrap_or_else(|| neutral_title(state, winner))
11477    } else {
11478        summary_title(summary).unwrap_or_else(|| {
11479            english_title(
11480                &state.instruction,
11481                PR_TITLE_MAX,
11482                state.config.graph.github_english(),
11483            )
11484            .unwrap_or_else(|| neutral_title(state, winner))
11485        })
11486    };
11487
11488    let mut body = String::new();
11489    let what = summary_without_title(summary);
11490    if !what.is_empty() {
11491        body.push_str("## Summary\n\n");
11492        body.push_str(&what);
11493        body.push_str("\n\n");
11494    }
11495
11496    // Review history uses structured facts across all rounds, never reports.
11497    body.push_str(&crate::pr_outcome::review_rounds(state));
11498    body.push_str(&crate::pr_outcome::final_verification(state, head));
11499    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11500
11501    let open = state.open_findings();
11502    if !open.is_empty() {
11503        body.push_str("## Open review findings\n\n");
11504        for f in &open {
11505            body.push_str(&format!(
11506                "- `{}` [{:?}] {}\n",
11507                f.id, f.severity, "Context retained in local review records."
11508            ));
11509        }
11510        body.push('\n');
11511    }
11512
11513    if let Some(fix) = fix
11514        && !fix.rejected.is_empty()
11515    {
11516        body.push_str("## Declined by the fixer\n\n");
11517        for r in &fix.rejected {
11518            body.push_str(&format!(
11519                "- `{}`: {}\n",
11520                r.id, "Reason retained in local review records."
11521            ));
11522        }
11523        body.push('\n');
11524    }
11525
11526    if review {
11527        // The review prompt is not the task; list what the branch carries.
11528        body.push_str("## Commits under review\n\n");
11529        if let Some(facts) = facts {
11530            let mut left = COMMITS_MAX_CHARS;
11531            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11532                let mut entry = format!("- {}\n", markdown_safe(subject));
11533                for l in markdown_safe(text).lines() {
11534                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11535                }
11536                if left == 0 {
11537                    body.push_str(&format!(
11538                        "- ... {} more commit(s)\n",
11539                        facts.commits.len() - i
11540                    ));
11541                    break;
11542                }
11543                if entry.len() > left {
11544                    // Even the first commit is cut: one huge body must not
11545                    // push the whole description past GitHub's limit.
11546                    let mut end = left;
11547                    while !entry.is_char_boundary(end) {
11548                        end -= 1;
11549                    }
11550                    entry.truncate(end);
11551                    entry.push_str("\n  ... (truncated)\n");
11552                    left = 0;
11553                } else {
11554                    left -= entry.len();
11555                }
11556                body.push_str(&entry);
11557            }
11558            if !facts.stat.trim().is_empty() {
11559                body.push_str(&format!(
11560                    "\n## Diff stat\n\n```\n{}\n```\n",
11561                    markdown_safe(facts.stat.trim())
11562                ));
11563            }
11564        } else {
11565            match &state.reviewed_commits {
11566                Some(subjects) => {
11567                    for s in subjects {
11568                        body.push_str(&format!("- {}\n", s.trim()));
11569                    }
11570                }
11571                None => {
11572                    // An older run kept only the prompt, with the commit list
11573                    // after its first paragraph.
11574                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11575                    body.push_str(rest.trim());
11576                    body.push('\n');
11577                }
11578            }
11579        }
11580    } else {
11581        let task = state.instruction.trim();
11582        let task = if task.is_empty() {
11583            "(empty task)"
11584        } else {
11585            task
11586        };
11587        body.push_str(&format!(
11588            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11589            task.replace("</details>", "&lt;/details&gt;")
11590        ));
11591    }
11592
11593    body.push_str(&format!(
11594        "\n---\nmagi:run/{} magi:candidate-{}\n",
11595        state.id,
11596        winner.to_ascii_lowercase()
11597    ));
11598
11599    PrMessage { title, body }
11600}
11601
11602/// Source imports have already been prepared from structured inputs. Legacy
11603/// inferred references must never instruct a seat to reconstruct those imports.
11604fn seeded_instruction(state: &RunState) -> String {
11605    let mut inputs = Vec::new();
11606    if let Some(start) = &state.sources.start_from {
11607        inputs.push(format!(
11608            "Start from `{}` at `{}` (including ancestry).",
11609            start.token, start.sha
11610        ));
11611    }
11612    for pick in &state.sources.pick_commits {
11613        inputs.push(format!(
11614            "Pick `{}` at `{}` (single commit).",
11615            pick.token, pick.sha
11616        ));
11617    }
11618    if inputs.is_empty() {
11619        state.instruction.clone()
11620    } else {
11621        format!(
11622            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11623            state.instruction,
11624            inputs.join("\n")
11625        )
11626    }
11627}
11628
11629/// Does the winner have no commits ahead of the base it would land on?
11630/// Any failure to find out reads as "not empty": the merge then behaves as it
11631/// always did rather than refusing on a guess.
11632async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11633    let base = &state.base_branch;
11634    let mut against = base.clone();
11635    if mode == MergeMode::Pr {
11636        // A pull request lands on the remote's base, never the local branch
11637        // of the same name: if that cannot be read, "not empty" is the safe
11638        // answer.
11639        let remote = &state.config.merge.remote;
11640        let tracking = format!("{remote}/{base}");
11641        let fetched = git::fetch(repo, remote, base).await;
11642        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11643            against = tracking;
11644        } else {
11645            return false;
11646        }
11647    }
11648    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11649}
11650
11651/// Why nothing was opened for an empty winner, with what the task's own
11652/// references resolved to.
11653fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11654    let mut detail = format!(
11655        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11656         nothing to open a pull request for"
11657    );
11658    detail.push_str(
11659        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11660    );
11661    detail
11662}
11663
11664/// What the `Pr` merge does once it knows whether the branch already has an
11665/// open pull request.
11666#[derive(Debug, PartialEq, Eq)]
11667enum PrPlan {
11668    Create,
11669    Adopt { url: String, title: String },
11670    Stop(String),
11671}
11672
11673/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11674/// a failed lookup -> stop with the real reason. Never guesses.
11675fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11676    match found {
11677        Ok(land::OpenPr::None) => PrPlan::Create,
11678        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11679        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11680            "several open pull requests exist for this branch, not picking one: {}",
11681            urls.join(" ")
11682        )),
11683        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11684    }
11685}
11686
11687/// `gh pr create`, returning the PR url.
11688pub(crate) async fn gh_pr_create(
11689    cwd: &Path,
11690    base: &str,
11691    head: &str,
11692    title: &str,
11693    body: &str,
11694) -> Result<String> {
11695    let out = tokio::process::Command::new("gh")
11696        .args([
11697            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11698        ])
11699        .current_dir(cwd)
11700        .quiet()
11701        .stdin(std::process::Stdio::null())
11702        .output()
11703        .await
11704        .context("spawn gh")?;
11705    if out.status.success() {
11706        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11707    } else {
11708        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11709    }
11710}
11711
11712/// Never force deletion of uncommitted work, including unknown legacy trees.
11713async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11714    match std::fs::symlink_metadata(path) {
11715        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11716            // A previous cleanup may have removed the tree before a branch
11717            // deletion or state save failed. Unregister it if still present.
11718            git::worktree_remove_clean(repo, path).await?;
11719            return Ok(());
11720        }
11721        Err(e) => return Err(e.into()),
11722        Ok(_) => {}
11723    }
11724    if !git::is_clean(path).await? {
11725        bail!(
11726            "worktree has uncommitted work; refusing removal: {}",
11727            path.display()
11728        );
11729    }
11730    if !git::worktree_remove_clean(repo, path).await? {
11731        bail!("worktree removal refused: {}", path.display());
11732    }
11733    Ok(())
11734}
11735
11736/// Tear a run's worktrees and branches down.
11737///
11738/// `home` is where the updated `run.json` is saved (via
11739/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11740/// a housekeeping pass already has its own honest `home` handed to it, and
11741/// falling through to the global here would write back through whichever
11742/// directory some other process or test pinned into that `OnceLock` first,
11743/// not the one the caller actually resolved its `runs` and `state` from.
11744pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11745    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11746        bail!("refusing fold: {reason}");
11747    }
11748    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11749        bail!("candidate preservation incomplete; refusing cleanup");
11750    }
11751    // Serialize ordinary folding with the same starts/resumes as target-only
11752    // eviction. Otherwise a second janitor could remove the whole worktree
11753    // around a live output borrower even though target eviction was fenced.
11754    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11755    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11756    let _driver = crate::driver::maintenance(home, &state.id)?;
11757    let mut output_guards = Vec::new();
11758    for entry in std::fs::read_dir(state.worktree_root())
11759        .into_iter()
11760        .flatten()
11761        .flatten()
11762    {
11763        if entry.path().join(".git").is_file() {
11764            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11765        }
11766    }
11767    let repo = state.repo.clone();
11768    // CLI, Web and automatic cleanup all enter here, independently of
11769    // Runner::fold_losers. Protect deletion and shared config release too.
11770    let wait = repository_wait(state, None);
11771    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11772        Ok(lease) => lease,
11773        Err(e) => {
11774            state.save_under(home)?;
11775            return Err(e);
11776        }
11777    };
11778    // Confirmed merges dispose of their owned sources after durable evidence
11779    // capture. Preflight every source before removing anything.
11780    let _source_outputs = if state.status == RunStatus::Merged
11781        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11782    {
11783        verify_merged_fold_sources(state, drop_winner, home).await?
11784    } else {
11785        Vec::new()
11786    };
11787    let root = state.worktree_root();
11788    let winner = state.tally.as_ref().map(|t| t.winner);
11789    let mut removed = Vec::new();
11790
11791    for i in 0..state.candidates.len() {
11792        let c = state.candidates[i].clone();
11793        let is_winner = Some(c.label) == winner;
11794        if is_winner && !drop_winner {
11795            continue;
11796        }
11797        let existed = c.worktree.exists();
11798        // A vanished directory can still have a Git registration that keeps
11799        // its branch checked out. Remove that registration before the branch.
11800        remove_preserved_worktree(&repo, &c.worktree).await?;
11801        if existed {
11802            removed.push(c.worktree.to_string_lossy().into_owned());
11803        }
11804        // A branch handed to a later run (and its pull request) is not this
11805        // run's to delete.
11806        let handed_over = state.released_branches.contains(&c.branch);
11807        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11808            if !git::branch_delete(&repo, &c.branch).await? {
11809                bail!("candidate branch removal refused");
11810            }
11811            removed.push(c.branch.clone());
11812        }
11813        state.candidates[i].folded = true;
11814    }
11815
11816    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11817        let path = name.path();
11818        let keep = !drop_winner
11819            && winner.is_some_and(|w| {
11820                path.file_name()
11821                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11822            });
11823        if keep {
11824            continue;
11825        }
11826        remove_preserved_worktree(&repo, &path).await?;
11827        removed.push(path.to_string_lossy().into_owned());
11828    }
11829
11830    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11831    // judge worktrees, so once the loop above has cleared all of them out,
11832    // the parent is a bare directory nobody else was ever going to remove -
11833    // git only ever managed what was inside it. Left alone, one of these
11834    // accumulates per fully-folded run; the operator's own machine had 74.
11835    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11836    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11837    // so does anything a run never claimed that happens to share the bay.
11838    remove_if_empty(&root);
11839
11840    if state.enabled_worktree_config && drop_winner {
11841        // A release, not a raw disable: some sibling run in this repository
11842        // may still hold its own reference (see `git::acquire_worktree_config`),
11843        // and only the last release actually turns the setting back off.
11844        git::release_worktree_config(&repo).await.ok();
11845        state.enabled_worktree_config = false;
11846    }
11847    state.save_under(home)?;
11848    Ok(removed)
11849}
11850
11851async fn verify_merged_fold_sources(
11852    state: &RunState,
11853    drop_winner: bool,
11854    home: &Path,
11855) -> Result<Vec<std::fs::File>> {
11856    let winner = state.tally.as_ref().map(|t| t.winner);
11857    let mut heads = BTreeSet::new();
11858    for candidate in &state.candidates {
11859        if Some(candidate.label) == winner && !drop_winner {
11860            continue;
11861        }
11862        anyhow::ensure!(
11863            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11864            "candidate recovery source is outside the inventoried run root: {}",
11865            candidate.worktree.display()
11866        );
11867        match std::fs::symlink_metadata(&candidate.worktree) {
11868            Ok(_) => {}
11869            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11870            Err(error) => return Err(error.into()),
11871        }
11872        if !state.released_branches.contains(&candidate.branch)
11873            && git::branch_exists(&state.repo, &candidate.branch).await?
11874        {
11875            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11876        }
11877    }
11878    let mut cargo_guards = Vec::new();
11879    let entries = match std::fs::read_dir(state.worktree_root()) {
11880        Ok(entries) => Some(entries),
11881        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11882        Err(error) => return Err(error.into()),
11883    };
11884    for entry in entries.into_iter().flatten() {
11885        let path = entry?.path();
11886        let metadata = std::fs::symlink_metadata(&path)?;
11887        anyhow::ensure!(
11888            metadata.is_dir() && !metadata.file_type().is_symlink(),
11889            "non-directory or symlink recovery source retained: {}",
11890            path.display()
11891        );
11892        if !drop_winner
11893            && winner.is_some_and(|w| {
11894                path.file_name()
11895                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11896            })
11897        {
11898            continue;
11899        }
11900        anyhow::ensure!(
11901            path.join(".git").is_file(),
11902            "unregistered recovery source retained: {}",
11903            path.display()
11904        );
11905        anyhow::ensure!(
11906            git::git(
11907                &path,
11908                &[
11909                    "status",
11910                    "--porcelain",
11911                    "--untracked-files=all",
11912                    "--ignored",
11913                    "--",
11914                    ".",
11915                    ":(exclude)target"
11916                ]
11917            )
11918            .await?
11919            .is_empty(),
11920            "source changes or ignored recovery files retained: {}",
11921            path.display()
11922        );
11923        match std::fs::symlink_metadata(path.join("target")) {
11924            Ok(_) => {
11925                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11926            }
11927            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11928            Err(error) => return Err(error.into()),
11929        }
11930        for checkpoint in [
11931            "rebase-merge",
11932            "rebase-apply",
11933            "sequencer",
11934            "CHERRY_PICK_HEAD",
11935            "MERGE_HEAD",
11936            "REBASE_HEAD",
11937            "REVERT_HEAD",
11938            "BISECT_LOG",
11939            "AUTO_MERGE",
11940            "MERGE_MSG",
11941        ] {
11942            let location = git::git(
11943                &path,
11944                &[
11945                    "rev-parse",
11946                    "--path-format=absolute",
11947                    "--git-path",
11948                    checkpoint,
11949                ],
11950            )
11951            .await?;
11952            anyhow::ensure!(
11953                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11954                if e.kind() == std::io::ErrorKind::NotFound),
11955                "Git recovery checkpoint retained: {checkpoint}"
11956            );
11957        }
11958        heads.insert(git::rev_parse(&path, "HEAD").await?);
11959    }
11960    crate::fold_evidence::preserve(state, home, &heads).await?;
11961
11962    Ok(cargo_guards)
11963}
11964
11965/// Remove `dir` if it exists and has nothing in it.
11966///
11967/// Best-effort and silent by design: a directory that is not empty (a run
11968/// whose winner is still parked there, a stray file some other process left)
11969/// is exactly the case this must refuse, and a directory that is already gone
11970/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11971/// already refuses a non-empty directory, so the emptiness check below is
11972/// belt, not suspenders - it is what keeps this from ever attempting the
11973/// removal in the case that matters, rather than trusting `remove_dir`'s
11974/// error path to have no side effects if it ever changed.
11975fn remove_if_empty(dir: &Path) {
11976    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11977        std::fs::remove_dir(dir).ok();
11978    }
11979}
11980
11981/// Severity of the worst open finding in the last review round, for reporting.
11982pub fn worst_open(state: &RunState) -> Option<Severity> {
11983    state
11984        .reviews
11985        .last()?
11986        .reviews
11987        .iter()
11988        .flat_map(|r| r.findings.iter())
11989        .map(|f| f.severity)
11990        .max()
11991}
11992
11993/// `[worktree] setup` hides its products per worktree, which needs
11994/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11995/// worktrees, like the hook's; every place that runs setup calls this first.
11996async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11997    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11998        git::acquire_worktree_config(repo).await?;
11999        state.enabled_worktree_config = true;
12000    }
12001    Ok(())
12002}
12003
12004#[cfg(test)]
12005mod tests {
12006    #[tokio::test]
12007    async fn launcher_base_fetch_requires_remote_success() {
12008        let tmp = tempfile::tempdir().unwrap();
12009        let repo = tmp.path().join("repo");
12010        let remote = tmp.path().join("remote");
12011        std::fs::create_dir(&repo).unwrap();
12012        std::fs::create_dir(&remote).unwrap();
12013        git::git(&remote, &["init", "--bare", "-b", "main"])
12014            .await
12015            .unwrap();
12016        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12017        git::git(
12018            &repo,
12019            &[
12020                "-c",
12021                "user.name=Fixture",
12022                "-c",
12023                "user.email=fixture@example.invalid",
12024                "commit",
12025                "--allow-empty",
12026                "-m",
12027                "fixture",
12028            ],
12029        )
12030        .await
12031        .unwrap();
12032        git::git(
12033            &repo,
12034            &["remote", "add", "origin", remote.to_str().unwrap()],
12035        )
12036        .await
12037        .unwrap();
12038        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12039        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12040        assert_eq!(
12041            resolve_base(&repo, "main", "origin", None)
12042                .await
12043                .unwrap()
12044                .commit,
12045            head
12046        );
12047        #[cfg(unix)]
12048        {
12049            let policy: crate::config::Coordination = toml::from_str(
12050                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12051            ).unwrap();
12052            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12053            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12054            let held = crate::repo_lock::SeatLease::operation_with(
12055                &repo,
12056                Some(&policy),
12057                &crate::repo_lock::WaitPolicy::default(),
12058            )
12059            .await
12060            .unwrap();
12061            assert!(held.is_some());
12062            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12063            tokio::pin!(fetch);
12064            assert!(
12065                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12066                    .await
12067                    .is_err()
12068            );
12069            assert!(
12070                !repo.join(".git/FETCH_HEAD").exists(),
12071                "no fetch before admission"
12072            );
12073            drop(held);
12074            let fetched = fetch.await.unwrap();
12075            assert_eq!(fetched.commit, head);
12076            assert_eq!(fetched.remote, "origin");
12077            assert_eq!(fetched.branch, "main");
12078        }
12079        // A surviving tracking ref is not successful fetch evidence.
12080        git::git(
12081            &repo,
12082            &[
12083                "remote",
12084                "set-url",
12085                "origin",
12086                tmp.path().join("missing").to_str().unwrap(),
12087            ],
12088        )
12089        .await
12090        .unwrap();
12091        let error = resolve_base(&repo, "main", "origin", None)
12092            .await
12093            .unwrap_err()
12094            .to_string();
12095        assert!(error.contains("cannot read origin/main"));
12096        assert!(error.contains("refusing to branch off"));
12097    }
12098
12099    #[test]
12100    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12101        let leaked = vec!["chore(deps): update a crate".to_owned()];
12102        let own = "fix(daemon): apply a chosen action";
12103        assert!(should_retitle("", own, &leaked));
12104        assert!(should_retitle(
12105            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12106            own,
12107            &leaked
12108        ));
12109        assert!(should_retitle(
12110            "chore: land candidate A of run 1",
12111            own,
12112            &leaked
12113        ));
12114        assert!(should_retitle(
12115            "magi: candidate A (uncommitted work)",
12116            own,
12117            &leaked
12118        ));
12119        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12120        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12121        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12122    }
12123
12124    #[test]
12125    fn pr_merge_plan_creates_adopts_or_stops() {
12126        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12127        assert_eq!(
12128            pr_merge_plan(Ok(land::OpenPr::One {
12129                url: "u".into(),
12130                title: "t".into()
12131            })),
12132            PrPlan::Adopt {
12133                url: "u".into(),
12134                title: "t".into()
12135            }
12136        );
12137        let PrPlan::Stop(many) =
12138            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12139        else {
12140            panic!("many must stop");
12141        };
12142        assert!(many.contains('a') && many.contains('b'));
12143        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12144            panic!("a failed lookup must stop");
12145        };
12146        assert!(err.contains("bad token"));
12147    }
12148
12149    use super::*;
12150    use crate::run::GateStatus;
12151    use std::collections::BTreeMap;
12152    use std::time::Duration;
12153
12154    fn conductor() -> AgentSpec {
12155        AgentSpec {
12156            id: "conductor".to_owned(),
12157            kind: crate::config::AgentKind::Command,
12158            model: None,
12159            command: vec!["true".to_owned()],
12160            extra_args: Vec::new(),
12161            env: BTreeMap::new(),
12162            prompt_delivery: None,
12163            max_turn_secs: None,
12164        }
12165    }
12166
12167    fn spec(id: &str) -> AgentSpec {
12168        AgentSpec {
12169            id: id.to_owned(),
12170            kind: crate::config::AgentKind::Command,
12171            model: None,
12172            command: vec!["true".to_owned()],
12173            extra_args: Vec::new(),
12174            env: BTreeMap::new(),
12175            prompt_delivery: None,
12176            max_turn_secs: None,
12177        }
12178    }
12179
12180    fn ids(xs: &[&str]) -> BTreeSet<String> {
12181        xs.iter().map(|s| (*s).to_owned()).collect()
12182    }
12183
12184    #[test]
12185    fn pick_successor_skips_an_agent_another_seat_holds() {
12186        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12187        let roster = [spec("a"), spec("b"), spec("c")];
12188        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12189        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12190    }
12191
12192    #[test]
12193    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12194        // The other seat started on c but was handed to d; c is free again.
12195        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12196        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12197        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12198        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12199        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12200    }
12201
12202    #[test]
12203    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12204        let roster = [spec("a"), spec("b")];
12205        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12206        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12207        let carried = ids(&["b"]);
12208        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12209        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12210    }
12211
12212    #[test]
12213    fn pick_successor_returns_none_without_any_untried_successor() {
12214        let roster = [spec("a"), spec("b")];
12215        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12216        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12217        let carried = ids(&["a"]);
12218        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12219    }
12220
12221    #[test]
12222    fn pick_successor_rescue_avoids_another_seats_occupant() {
12223        // b is a carried failure and free; a is held by the other seat.
12224        let roster = [spec("a"), spec("b"), spec("c")];
12225        let carried = ids(&["b", "c"]);
12226        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12227        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12228    }
12229
12230    #[test]
12231    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12232        let roster = [spec("a"), spec("b"), spec("c")];
12233        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12234        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12235    }
12236
12237    #[test]
12238    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12239        let roster = [spec("a"), spec("b"), spec("c")];
12240        let failed = ids(&["a", "b", "c"]);
12241        // Rescue looks at the whole roster, once per id, then runs out.
12242        let mut tried = ids(&["b"]);
12243        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12244        assert_eq!(first.id, "a");
12245        tried.insert(first.id.clone());
12246        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12247        assert_eq!(second.id, "c");
12248        tried.insert(second.id.clone());
12249        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12250    }
12251
12252    #[test]
12253    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12254        let roster = [spec("a"), spec("b")];
12255        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12256        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12257    }
12258
12259    #[test]
12260    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12261        let roster = [spec("a"), spec("b"), spec("c")];
12262        let h = SeatHistory {
12263            failed: ids(&["a"]),
12264            last_ok: Some("b".to_owned()),
12265            last_fail: None,
12266        };
12267        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12268        // A last answerer that left the roster, or later failed, is ignored.
12269        let gone = SeatHistory {
12270            last_ok: Some("zzz".to_owned()),
12271            ..h.clone()
12272        };
12273        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12274        let failed = SeatHistory {
12275            failed: ids(&["a", "b"]),
12276            last_ok: Some("b".to_owned()),
12277            last_fail: None,
12278        };
12279        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12280    }
12281
12282    #[test]
12283    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12284        let first = SeatState::new("review-1", "alpha", 7);
12285        let next = handover_seat("review-1", "gamma", 7);
12286        assert_ne!(first.claude_session, next.claude_session);
12287    }
12288
12289    #[test]
12290    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12291        let mut state = state_with_summary("x", "y");
12292        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12293        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12294        assert_ne!(a.claude_session, b.claude_session);
12295    }
12296
12297    #[test]
12298    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12299        let roster = [spec("a"), spec("b")];
12300        let h = SeatHistory {
12301            failed: ids(&["a", "b"]),
12302            ..SeatHistory::default()
12303        };
12304        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12305        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12306        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12307    }
12308
12309    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12310    // fallback loop depends on to terminate: it must walk forward from the
12311    // seat's own position, never restart at the front of the roster, and it
12312    // must never hand back an id already tried, however many times that id
12313    // happens to appear.
12314
12315    #[test]
12316    fn failure_signature_ignores_numbers_and_paths() {
12317        assert_eq!(
12318            failure_signature("exited with Some(2) and no usable output"),
12319            failure_signature("exited with Some(137) and no usable output")
12320        );
12321        assert_eq!(
12322            failure_signature("process exited with no usable output (exit code 2)"),
12323            failure_signature("process exited with no usable output (exit code 137)")
12324        );
12325        for (old, new) in [
12326            (
12327                "exited with Some(2) and no usable output",
12328                "process exited with no usable output (exit code 2)",
12329            ),
12330            (
12331                "exited with None and no usable output",
12332                "process exited with no usable output (exit code unavailable)",
12333            ),
12334        ] {
12335            let previous = FailClass::Other(failure_signature(old));
12336            let current = FailClass::Other(failure_signature(new));
12337            assert_eq!(previous, current, "resumption retains the failure class");
12338            assert!(!should_hand_over(Some(&previous), &current));
12339        }
12340        assert_eq!(
12341            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12342            failure_signature("cannot open /var/x.txt: denied")
12343        );
12344        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12345    }
12346
12347    #[test]
12348    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12349        let other = FailClass::Other("x".into());
12350        assert!(should_hand_over(None, &FailClass::Quota));
12351        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12352        assert!(should_hand_over(
12353            Some(&FailClass::Timeout),
12354            &FailClass::Timeout
12355        ));
12356        assert!(should_hand_over(None, &other));
12357        assert!(!should_hand_over(Some(&other), &other));
12358        assert!(should_hand_over(
12359            Some(&other),
12360            &FailClass::Other("y".into())
12361        ));
12362        // A quota or timeout in between ends the run of identical failures.
12363        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12364        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12365    }
12366
12367    #[test]
12368    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12369        let a = SeatState::new("judge-1", "alpha", 7);
12370        let b = handover_seat("judge-1", "beta", 7);
12371        assert_ne!(a.claude_session, b.claude_session);
12372        assert_eq!(b.turns, 0);
12373    }
12374
12375    #[test]
12376    fn a_timeout_is_classified_apart_from_other_failures() {
12377        assert_eq!(
12378            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12379            Some(FailClass::Timeout)
12380        );
12381        assert!(matches!(
12382            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12383            Some(FailClass::Other(_))
12384        ));
12385    }
12386
12387    #[test]
12388    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12389        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12390        let tried = BTreeSet::from(["beta".to_owned()]);
12391        // beta sits at index 1; the next candidate is gamma, never alpha —
12392        // which is very likely a different candidate slot's own agent.
12393        let next = next_untried_in_roster(&roster, 1, &tried);
12394        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12395    }
12396
12397    #[test]
12398    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12399        let roster = vec![spec("alpha"), spec("beta")];
12400        let tried = BTreeSet::from(["beta".to_owned()]);
12401        // beta is the roster's last entry: nothing follows it, and alpha —
12402        // earlier in the roster, almost certainly a different candidate
12403        // slot's own agent — must not be reached by wrapping back to it.
12404        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12405    }
12406
12407    #[test]
12408    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12409        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12410        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12411        // beta (index 1) and gamma (index 2, the only entry after it) have
12412        // both been tried; alpha (index 0) never has, but it comes before
12413        // beta's own position, so there is nothing further for this seat.
12414        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12415    }
12416
12417    #[test]
12418    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12419        let roster = vec![spec("a"), spec("a"), spec("b")];
12420        let tried = BTreeSet::from(["a".to_owned()]);
12421        let next = next_untried_in_roster(&roster, 0, &tried);
12422        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12423    }
12424
12425    #[test]
12426    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12427        let roster = vec![spec("a"), spec("b")];
12428        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12429        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12430    }
12431
12432    #[test]
12433    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12434        let dir = tempfile::tempdir().unwrap();
12435        let bay = dir.path().join("ffff");
12436
12437        // Not there yet: nothing to do, nothing to panic on.
12438        remove_if_empty(&bay);
12439        assert!(!bay.exists());
12440
12441        // Something still inside - the winner's worktree, or a stray file -
12442        // keeps the directory standing.
12443        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12444        remove_if_empty(&bay);
12445        assert!(bay.exists(), "non-empty directory must survive");
12446
12447        // Once the last entry is gone, so is the directory itself.
12448        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12449        remove_if_empty(&bay);
12450        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12451    }
12452
12453    // `round_is_clean` is the exact decision this task fixed: a round with a
12454    // seat that never answered must not read the same as a round every seat
12455    // actually reviewed. These are deterministic and process-free by design —
12456    // the equivalent end-to-end check (a real reviewer timing out under a
12457    // live graph run) is a genuine race against wall-clock contention, and a
12458    // spawn slow enough to blow even a generous budget under a loaded test
12459    // run must not turn this specific regression check flaky.
12460
12461    #[test]
12462    fn a_full_panel_that_found_nothing_is_clean() {
12463        assert!(round_is_clean(
12464            0,
12465            true,
12466            2,
12467            2,
12468            0,
12469            IncompleteReviewPolicy::Block
12470        ));
12471    }
12472
12473    #[test]
12474    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12475        assert!(!round_is_clean(
12476            0,
12477            true,
12478            1,
12479            2,
12480            0,
12481            IncompleteReviewPolicy::Block
12482        ));
12483    }
12484
12485    #[test]
12486    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12487        assert!(!round_is_clean(
12488            1,
12489            true,
12490            1,
12491            2,
12492            0,
12493            IncompleteReviewPolicy::Warn
12494        ));
12495    }
12496
12497    #[test]
12498    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12499        assert!(round_is_clean(
12500            0,
12501            true,
12502            1,
12503            2,
12504            0,
12505            IncompleteReviewPolicy::Warn
12506        ));
12507    }
12508
12509    #[test]
12510    fn a_full_panel_with_an_open_finding_is_not_clean() {
12511        assert!(!round_is_clean(
12512            1,
12513            true,
12514            2,
12515            2,
12516            0,
12517            IncompleteReviewPolicy::Block
12518        ));
12519    }
12520
12521    #[test]
12522    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12523        assert!(!round_is_clean(
12524            0,
12525            false,
12526            2,
12527            2,
12528            0,
12529            IncompleteReviewPolicy::Block
12530        ));
12531    }
12532
12533    // The stall this task closes: under the default `block` policy, a seat
12534    // missing only because it was rate limited must not force a wait for a
12535    // session limit that will not lift by the next round. `round_is_clean`
12536    // is where that quorum carve-out lives; the review loop around it never
12537    // changes what a reviewer's vote or a finding's severity means.
12538
12539    #[test]
12540    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12541        // 1 of 2 answered, and the one missing was quota'd — the exact
12542        // "review-2 rate limited (quota)" shape from the field report.
12543        assert!(round_is_clean(
12544            0,
12545            true,
12546            1,
12547            2,
12548            1,
12549            IncompleteReviewPolicy::Block
12550        ));
12551    }
12552
12553    #[test]
12554    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12555        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12556        // not a quota loss (`quota_missing` stays 0) — worth another try.
12557        assert!(!round_is_clean(
12558            0,
12559            true,
12560            1,
12561            2,
12562            0,
12563            IncompleteReviewPolicy::Block
12564        ));
12565    }
12566
12567    #[test]
12568    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12569        assert!(!round_is_clean(
12570            1,
12571            true,
12572            1,
12573            2,
12574            1,
12575            IncompleteReviewPolicy::Block
12576        ));
12577        assert!(!round_is_clean(
12578            0,
12579            false,
12580            1,
12581            2,
12582            1,
12583            IncompleteReviewPolicy::Block
12584        ));
12585    }
12586
12587    #[test]
12588    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12589        // Every seat quota'd, nobody answered: there is no panel to decide
12590        // on, so this must fall through to the existing block-and-retry
12591        // fallback rather than call an unreviewed patch clean.
12592        assert!(!round_is_clean(
12593            0,
12594            true,
12595            0,
12596            2,
12597            2,
12598            IncompleteReviewPolicy::Block
12599        ));
12600    }
12601
12602    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12603        CommandOutcome {
12604            command: "test".to_owned(),
12605            code,
12606            output_tail: String::new(),
12607            duration_ms: 0,
12608            resource_blocked,
12609        }
12610    }
12611
12612    #[test]
12613    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12614        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12615        assert!(
12616            !verify_inconclusive(&[outcome(Some(1), false)]),
12617            "an ordinary failure is still evidence about the patch"
12618        );
12619        assert!(verify_inconclusive(&[outcome(None, true)]));
12620        assert!(
12621            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12622            "one inconclusive outcome taints the whole batch"
12623        );
12624        assert!(!verify_inconclusive(&[]));
12625    }
12626
12627    #[tokio::test]
12628    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12629        // Alive for the first two checks, then dead - confirms the loop
12630        // actually re-polls rather than deciding once and sleeping out the
12631        // ceiling regardless.
12632        let calls = std::sync::atomic::AtomicUsize::new(0);
12633        let started = Instant::now();
12634        wait_for_pids_with(
12635            &[123],
12636            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12637            Duration::from_millis(5),
12638            Duration::from_secs(5),
12639        )
12640        .await;
12641        assert!(
12642            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12643            "must keep checking rather than deciding on the first answer"
12644        );
12645        assert!(
12646            started.elapsed() < Duration::from_secs(1),
12647            "must return the moment it is confirmed dead, not wait out the ceiling"
12648        );
12649    }
12650
12651    #[tokio::test]
12652    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12653        let started = Instant::now();
12654        wait_for_pids_with(
12655            &[123],
12656            |_| true, // never reports dead
12657            Duration::from_millis(5),
12658            Duration::from_millis(30),
12659        )
12660        .await;
12661        let elapsed = started.elapsed();
12662        assert!(
12663            elapsed >= Duration::from_millis(30),
12664            "must not give up before its own ceiling: {elapsed:?}"
12665        );
12666        assert!(
12667            elapsed < Duration::from_secs(1),
12668            "must not wait past its own ceiling either: {elapsed:?}"
12669        );
12670    }
12671
12672    #[tokio::test]
12673    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12674        let started = Instant::now();
12675        wait_for_pids_with(
12676            &[],
12677            |_| true,
12678            Duration::from_secs(5),
12679            Duration::from_secs(5),
12680        )
12681        .await;
12682        assert!(
12683            started.elapsed() < Duration::from_millis(200),
12684            "an empty pid list has nothing to confirm"
12685        );
12686    }
12687
12688    // `review_conclusion` is the exact decision the review hand-off task
12689    // fixed: a round budget spent (or a tree that stopped moving) must not
12690    // collapse into `Blocked` regardless of what verification actually
12691    // said. Deterministic and process-free for the same reason the
12692    // `round_is_clean` family above is.
12693    fn review_round(
12694        clean: bool,
12695        blocking: usize,
12696        answered: usize,
12697        expected: usize,
12698        progressed: bool,
12699        e2e_ok: bool,
12700    ) -> ReviewRound {
12701        ReviewRound {
12702            round: 1,
12703            head: "h".to_owned(),
12704            verified_head: None,
12705            verified_at: None,
12706            reviews: Vec::new(),
12707            e2e: vec![CommandOutcome {
12708                command: "test".to_owned(),
12709                code: Some(if e2e_ok { 0 } else { 1 }),
12710                output_tail: String::new(),
12711                duration_ms: 0,
12712                resource_blocked: false,
12713            }],
12714            verify_retried: false,
12715            e2e_deferred: false,
12716            e2e_defer_reason: None,
12717            fix: None,
12718            blocking,
12719            answered,
12720            expected,
12721            clean,
12722            progressed,
12723            vote_split: false,
12724            reconsideration: Vec::new(),
12725            verdict: None,
12726        }
12727    }
12728
12729    #[test]
12730    fn review_conclusion_is_none_when_nothing_has_run() {
12731        assert_eq!(review_conclusion(&[], 3), None);
12732    }
12733
12734    #[test]
12735    fn review_conclusion_is_none_while_rounds_remain() {
12736        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12737        assert_eq!(review_conclusion(&rounds, 3), None);
12738    }
12739
12740    #[test]
12741    fn review_conclusion_is_gating_once_a_round_is_clean() {
12742        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12743        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12744    }
12745
12746    #[test]
12747    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12748        let rounds = vec![
12749            review_round(false, 1, 2, 2, true, true),
12750            review_round(false, 1, 2, 2, true, true),
12751        ];
12752        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12753    }
12754
12755    #[test]
12756    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12757        let rounds = vec![
12758            review_round(false, 1, 2, 2, true, true),
12759            review_round(false, 1, 2, 2, true, false),
12760        ];
12761        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12762    }
12763
12764    #[test]
12765    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12766        // Magi never got a command to run against this round's own head — a
12767        // resource-blocked attempt, not a red one — so this must never
12768        // settle on `Blocked` the way a genuine e2e failure would. `None`
12769        // here is what tells `Runner::review_loop` to retry the check
12770        // itself rather than trust this cheap recomputation with a verdict
12771        // it cannot actually produce.
12772        let mut blocked = review_round(false, 1, 2, 2, true, false);
12773        blocked.e2e[0].resource_blocked = true;
12774        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12775        assert_eq!(review_conclusion(&rounds, 2), None);
12776    }
12777
12778    #[test]
12779    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12780        // Missing input, not a verified tree — never a hand-off candidate.
12781        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12782        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12783    }
12784
12785    #[test]
12786    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12787        let rounds = vec![
12788            review_round(false, 1, 2, 2, false, true),
12789            review_round(false, 1, 2, 2, false, true),
12790        ];
12791        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12792    }
12793
12794    fn secs(n: u64) -> Duration {
12795        Duration::from_secs(n)
12796    }
12797
12798    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12799    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12800    #[tokio::test]
12801    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12802        ask_test_home();
12803        let tmp = tempfile::tempdir().unwrap();
12804        let repo = tmp.path().join("repo");
12805        std::fs::create_dir_all(&repo).unwrap();
12806        init_repo(&repo);
12807        let g = |args: &[&str]| {
12808            let out = std::process::Command::new("git")
12809                .args(args)
12810                .current_dir(&repo)
12811                .quiet()
12812                .output()
12813                .unwrap();
12814            assert!(
12815                out.status.success(),
12816                "{}",
12817                String::from_utf8_lossy(&out.stderr)
12818            );
12819        };
12820        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12821        g(&["checkout", "-b", "historical"]);
12822        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12823        g(&["add", "."]);
12824        g(&["commit", "-m", "old alternative"]);
12825        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12826        g(&["checkout", "-b", "magi/task-selected", &base]);
12827        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12828        g(&["add", "."]);
12829        g(&["commit", "-m", "measurement infrastructure"]);
12830        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12831        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12832        g(&["add", "."]);
12833        g(&["commit", "-m", "dependent implementation"]);
12834        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12835        g(&["checkout", "main"]);
12836        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12837        g(&["add", "."]);
12838        g(&["commit", "-m", "unrelated base change"]);
12839        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12840        let origin = tmp.path().join("origin.git");
12841        g(&[
12842            "clone",
12843            "--bare",
12844            repo.to_str().unwrap(),
12845            origin.to_str().unwrap(),
12846        ]);
12847        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12848        g(&["fetch", "origin"]);
12849        let selection = crate::pr_recovery::Selection {
12850            url: "https://example.test/project/pull/1".into(),
12851            branch: "magi/task-selected".into(),
12852            head: head.clone(),
12853            source_head: None,
12854            source_run: None,
12855            source_proof: None,
12856            disposition: Some(crate::pr_recovery::Disposition::Review),
12857            instruction: "preserve measured behavior".into(),
12858            approved_at: jiff::Timestamp::now(),
12859        };
12860        let mut template = runner_at(RunStatus::Reviewing);
12861        template.state.repo = repo.clone();
12862        template.state.base_commit = current.clone();
12863        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12864        template.state.origin = Some(crate::run::Origin::queue("selected"));
12865        template.state.pr_head = Some(selection.branch.clone());
12866        template.state.pr_recovery = Some(selection.clone());
12867        let mut runner =
12868            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12869                .await
12870                .unwrap();
12871        let wt = runner.state.winner().unwrap().worktree.clone();
12872        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12873        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12874        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12875        assert!(runner.state.seeds.is_empty());
12876        assert_eq!(runner.state.instruction, "preserve measured behavior");
12877        runner.sync_to_base().await.unwrap();
12878        assert_eq!(
12879            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12880            "prerequisite"
12881        );
12882        assert_eq!(
12883            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12884            "measured implementation"
12885        );
12886        assert_eq!(
12887            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12888            "current main change"
12889        );
12890        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12891        assert_eq!(
12892            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12893            head
12894        );
12895
12896        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12897        let mut partial = runner_at(RunStatus::Reviewing);
12898        partial.state.repo = repo.clone();
12899        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12900        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12901            run: Some("exit 1".into()),
12902            ..Default::default()
12903        }];
12904        let selection = runner.state.pr_recovery.clone().unwrap();
12905        partial.state.pr_recovery = Some(selection.clone());
12906        partial.state.pr_head = Some(selection.branch.clone());
12907        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12908        let id = partial.state.id.clone();
12909        let error = match Runner::open_selected(
12910            &repo,
12911            partial.state,
12912            partial.roles,
12913            1,
12914            runner.state.base_commit.clone(),
12915            selection,
12916        )
12917        .await
12918        {
12919            Ok(_) => panic!("setup must fail"),
12920            Err(e) => e,
12921        };
12922        assert_eq!(
12923            error
12924                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12925                .unwrap()
12926                .run,
12927            id
12928        );
12929        let failed = RunState::load(&id).unwrap();
12930        assert_eq!(failed.status, RunStatus::Blocked);
12931        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12932        assert!(Runner::resume(&id).is_err());
12933        runner.state.status = RunStatus::Ready;
12934        runner.state.driver_exited = true;
12935        runner.state.save().unwrap();
12936        let mut retry = runner_at(RunStatus::Reviewing);
12937        retry.state.repo = repo.clone();
12938        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12939        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12940        reselected.approved_at = jiff::Timestamp::now();
12941        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12942        retry.state.pr_head = Some(reselected.branch.clone());
12943        retry.state.pr_recovery = Some(reselected.clone());
12944        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12945            .await
12946            .unwrap();
12947        let restarted = Runner::open_selected(
12948            &repo,
12949            retry.state,
12950            retry.roles,
12951            1,
12952            runner.state.base_commit.clone(),
12953            reselected,
12954        )
12955        .await
12956        .unwrap();
12957        assert_eq!(
12958            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12959                .await
12960                .unwrap(),
12961            head
12962        );
12963        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12964        assert_eq!(
12965            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12966            head
12967        );
12968    }
12969
12970    #[tokio::test]
12971    #[cfg(unix)]
12972    async fn fallback_and_interrupted_restoration_require_coordination() {
12973        use std::os::fd::AsRawFd as _;
12974        ask_test_home();
12975        let tmp = tempfile::tempdir().unwrap();
12976        let mut runner = registered_runner(tmp.path()).await;
12977        runner.state.config.graph.timeout_resource_wait = 1;
12978        let winner = runner.state.candidates[0].clone();
12979        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12980        let leftover = winner.worktree.join("leftover");
12981        std::fs::write(&leftover, "untrusted edits").unwrap();
12982        let next = tmp.path().join("next-fixer");
12983        let mut failed = spec("first");
12984        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12985        let mut successor = spec("second");
12986        successor.command = vec![
12987            "sh".into(),
12988            "-c".into(),
12989            format!("touch '{}'; echo done", next.display()),
12990        ];
12991        let attempts = vec![
12992            (failed.clone(), "first".into()),
12993            (successor.clone(), "second".into()),
12994        ];
12995        let lock_path = runner.state.repo.join(".git/updater.lock");
12996        let external = std::fs::File::create(&lock_path).unwrap();
12997        assert_eq!(
12998            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12999            0
13000        );
13001        let artifacts = tmp.path().join("artifacts");
13002        let build = |spec, seat| SeatJob {
13003            spec,
13004            seat,
13005            cwd: winner.worktree.clone(),
13006            prompt: "fixture".into(),
13007            timeout: Duration::from_secs(10),
13008            allow_write: true,
13009            sessions: false,
13010            artifacts: artifacts.clone(),
13011            stem: "fix".into(),
13012            handover: None,
13013        };
13014        let (_, _, out) = runner
13015            .ask_fixer_among(
13016                attempts.clone(),
13017                "graft",
13018                None,
13019                Some((&winner.worktree, &head)),
13020                build,
13021            )
13022            .await;
13023        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13024        assert!(leftover.exists(), "contention must prevent clean");
13025        assert!(!next.exists());
13026        assert!(runner.state.handovers.is_empty());
13027        // Early interrupted-repair rollback uses the same protected route.
13028        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13029            from_run: "earlier".into(),
13030            generation: 1,
13031            mode: crate::recovery::Mode::VerifyRepair,
13032            reasons: Vec::new(),
13033            reviewed_head: None,
13034            rounds: Vec::new(),
13035            open_findings: Vec::new(),
13036            evidence: None,
13037            criteria_inherited: true,
13038            repair: crate::recovery::Repair::Started {
13039                before: head.clone(),
13040            },
13041        });
13042        assert!(
13043            runner
13044                .recovery_repair(&winner)
13045                .await
13046                .unwrap_err()
13047                .is::<crate::repo_lock::ResourceBlocked>()
13048        );
13049        assert!(matches!(
13050            runner.state.recovery.as_ref().unwrap().repair,
13051            crate::recovery::Repair::Started { .. }
13052        ));
13053        assert!(leftover.exists());
13054        drop(external);
13055        runner.recovery_repair(&winner).await.unwrap();
13056        assert!(!leftover.exists());
13057        assert!(matches!(
13058            runner.state.recovery.as_ref().unwrap().repair,
13059            crate::recovery::Repair::Done { .. }
13060        ));
13061        // A hard lock-open failure also leaves HEAD and untracked work intact.
13062        std::fs::remove_file(&lock_path).unwrap();
13063        std::fs::create_dir(&lock_path).unwrap();
13064        std::fs::write(&leftover, "untrusted edits").unwrap();
13065        let (_, _, out) = runner
13066            .ask_fixer_among(
13067                attempts,
13068                "graft",
13069                None,
13070                Some((&winner.worktree, &head)),
13071                build,
13072            )
13073            .await;
13074        assert!(matches!(out, AgentOutcome::Failed(_)));
13075        assert!(leftover.exists());
13076        assert!(!next.exists());
13077        assert_eq!(
13078            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13079            head
13080        );
13081    }
13082
13083    #[cfg(unix)]
13084    async fn registered_runner(root: &Path) -> Runner {
13085        ask_test_home();
13086        let repo = root.join("repo");
13087        std::fs::create_dir_all(&repo).unwrap();
13088        init_repo(&repo);
13089        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13090        let mut runner = runner_at(RunStatus::Prep);
13091        runner.state.repo = repo.clone();
13092        runner.state.config.coordination = crate::config::test_coordination();
13093        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13094        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13095        runner.state.config.graph.advise = false;
13096        runner.roles.implementers = vec![spec("implement")];
13097        runner.roles.judges = vec![spec("judge")];
13098        runner.prep().await.unwrap();
13099        runner
13100    }
13101
13102    #[tokio::test]
13103    #[cfg(unix)]
13104    async fn parked_implementer_persists_completed_work() {
13105        for registered in [false, true] {
13106            let tmp = tempfile::tempdir().unwrap();
13107            let mut runner = registered_runner(tmp.path()).await;
13108            if !registered {
13109                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13110            }
13111            let marker = tmp.path().join("started");
13112            let release = tmp.path().join("release");
13113            let mut implementer = spec("implement");
13114            implementer.command = vec![
13115                "sh".into(),
13116                "-c".into(),
13117                format!(
13118                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13119                    marker.display(),
13120                    release.display()
13121                ),
13122            ];
13123            runner.roles.implementers = vec![implementer.clone()];
13124            runner.state.config.agents = vec![implementer];
13125            let pause = runner.pause.clone();
13126            let (result, ()) = tokio::join!(runner.implement(), async {
13127                tokio::time::timeout(Duration::from_secs(10), async {
13128                    while !marker.exists() {
13129                        tokio::time::sleep(Duration::from_millis(10)).await;
13130                    }
13131                })
13132                .await
13133                .unwrap();
13134                pause.park();
13135                std::fs::write(&release, "go").unwrap();
13136            });
13137            result.unwrap();
13138            let candidate = &runner.state.candidates[0];
13139            assert!(candidate.commits > 0, "{candidate:?}");
13140            assert!(candidate.summary.contains("saved-result"));
13141            assert_eq!(candidate.files, 1);
13142            let before = candidate.commits;
13143            runner.implement().await.unwrap();
13144            assert_eq!(runner.state.candidates[0].commits, before);
13145        }
13146    }
13147
13148    #[tokio::test]
13149    #[cfg(unix)]
13150    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13151        let tmp = tempfile::tempdir().unwrap();
13152        let mut runner = registered_runner(tmp.path()).await;
13153        let winner = runner.state.candidates[0].clone();
13154        let cache = tmp.path().join("cache");
13155        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13156        runner.state.config.graph.timeout_resource_wait = 0;
13157        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13158        let crate::cache::AcquireOutcome::Acquired(held) =
13159            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13160        else {
13161            panic!("cache must be free")
13162        };
13163        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13164            from_run: "earlier".into(),
13165            generation: 1,
13166            mode: crate::recovery::Mode::VerifyRepair,
13167            reasons: Vec::new(),
13168            reviewed_head: None,
13169            rounds: Vec::new(),
13170            open_findings: Vec::new(),
13171            evidence: None,
13172            criteria_inherited: true,
13173            repair: crate::recovery::Repair::NotStarted,
13174        });
13175        runner.recovery_repair(&winner).await.unwrap();
13176        assert!(matches!(
13177            runner.state.recovery.as_ref().unwrap().repair,
13178            crate::recovery::Repair::WaitingForRestore { .. }
13179        ));
13180        drop(held);
13181        runner.state.resource_block = None;
13182        runner.state.config.graph.timeout_resource_wait = 1;
13183        runner.recovery_repair(&winner).await.unwrap();
13184        assert!(matches!(
13185            runner.state.recovery.as_ref().unwrap().repair,
13186            crate::recovery::Repair::Done { .. }
13187        ));
13188        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13189    }
13190
13191    #[tokio::test]
13192    #[cfg(unix)]
13193    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13194        use std::os::fd::AsRawFd as _;
13195        for graft in [false, true] {
13196            let tmp = tempfile::tempdir().unwrap();
13197            let mut runner = registered_runner(tmp.path()).await;
13198            runner.state.config.graph.timeout_resource_wait = 1;
13199            let winner = runner.state.candidates[0].clone();
13200            let marker = tmp.path().join("fixer-started");
13201            let mut fixer = spec("fixer");
13202            fixer.command = vec![
13203                "sh".into(),
13204                "-c".into(),
13205                format!("echo started >> '{}'; echo done", marker.display()),
13206            ];
13207            runner.roles.fixer = Some(vec![fixer]);
13208            if graft {
13209                let mut loser = winner.clone();
13210                loser.label = 'B';
13211                loser.index = 1;
13212                runner.state.candidates.push(loser);
13213                runner.state.config.graph.graft_runner_up = true;
13214                runner.state.tally = Some(Tally {
13215                    first_choice: BTreeMap::new(),
13216                    borda: BTreeMap::new(),
13217                    winner: winner.label,
13218                    rankings: 1,
13219                    unanimous_initial: true,
13220                    deliberated: false,
13221                    changed_votes: 0,
13222                    unanimous_final: true,
13223                    tie_break: None,
13224                    judges: 1,
13225                    present: 1,
13226                    quorum: 1,
13227                    met_quorum: true,
13228                    uncontested: None,
13229                });
13230                runner.state.judgements.push(crate::run::Judgement {
13231                    judge: 1,
13232                    seat: "judge-1".into(),
13233                    agent: "judge".into(),
13234                    ranking: vec![winner.label, 'B'],
13235                    reasons: BTreeMap::new(),
13236                    confidence: None,
13237                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13238                    order: vec![0, 1],
13239                    failed: None,
13240                    duration_ms: 0,
13241                });
13242            } else {
13243                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13244                    from_run: "earlier".into(),
13245                    generation: 1,
13246                    mode: crate::recovery::Mode::VerifyRepair,
13247                    reasons: Vec::new(),
13248                    reviewed_head: None,
13249                    rounds: Vec::new(),
13250                    open_findings: Vec::new(),
13251                    evidence: None,
13252                    criteria_inherited: true,
13253                    repair: crate::recovery::Repair::NotStarted,
13254                });
13255            }
13256            let reader = crate::repo_lock::SeatLease::for_checkout(
13257                &winner.worktree,
13258                false,
13259                crate::config::test_coordination().as_ref(),
13260            )
13261            .await
13262            .unwrap();
13263            let pause = runner.pause.clone();
13264            let (result, ()) = tokio::join!(
13265                async {
13266                    if graft {
13267                        runner.graft().await
13268                    } else {
13269                        runner.recovery_repair(&winner).await
13270                    }
13271                },
13272                async {
13273                    tokio::time::sleep(Duration::from_millis(100)).await;
13274                    pause.park();
13275                }
13276            );
13277            result.unwrap();
13278            assert!(!marker.exists(), "a blocked seat must not spawn");
13279            // Exercise the persisted state, as a binary replacement/resume does.
13280            runner.state = RunState::load(&runner.state.id).unwrap();
13281            if graft {
13282                assert_eq!(
13283                    runner.state.graft.as_ref().unwrap().outcome,
13284                    GraftOutcome::WaitingForRestore
13285                );
13286            } else {
13287                assert!(matches!(
13288                    runner.state.recovery.as_ref().unwrap().repair,
13289                    crate::recovery::Repair::WaitingForRestore { .. }
13290                ));
13291            }
13292            drop(reader);
13293            runner.pause = Pause::new();
13294            runner.state.resource_block = None;
13295            // Deferred restoration must still respect an external lock owner.
13296            let external =
13297                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13298            assert_eq!(
13299                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13300                0
13301            );
13302            let leftover = winner.worktree.join("leftover");
13303            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13304            let result = if graft {
13305                runner.graft().await
13306            } else {
13307                runner.recovery_repair(&winner).await
13308            };
13309            assert!(
13310                result
13311                    .unwrap_err()
13312                    .is::<crate::repo_lock::ResourceBlocked>()
13313            );
13314            assert!(leftover.exists());
13315            assert!(!marker.exists());
13316            drop(external);
13317            runner.state.resource_block = None;
13318            let result = if graft {
13319                runner.graft().await
13320            } else {
13321                runner.recovery_repair(&winner).await
13322            };
13323            result.unwrap();
13324            assert!(!leftover.exists(), "restore must precede retry");
13325            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13326            if graft {
13327                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13328            } else {
13329                assert!(matches!(
13330                    runner.state.recovery.as_ref().unwrap().repair,
13331                    crate::recovery::Repair::Done { .. }
13332                ));
13333            }
13334        }
13335    }
13336
13337    #[tokio::test]
13338    #[cfg(unix)]
13339    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13340        use std::os::fd::AsRawFd as _;
13341        let tmp = tempfile::tempdir().unwrap();
13342        let mut runner = registered_runner(tmp.path()).await;
13343        runner.state.config.graph.timeout_resource_wait = 1;
13344        let winner = runner.state.candidates[0].clone();
13345        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13346        runner.state.tally = Some(Tally {
13347            first_choice: BTreeMap::new(),
13348            borda: BTreeMap::new(),
13349            winner: winner.label,
13350            rankings: 0,
13351            unanimous_initial: false,
13352            deliberated: false,
13353            changed_votes: 0,
13354            unanimous_final: false,
13355            tie_break: None,
13356            judges: 0,
13357            present: 0,
13358            quorum: 0,
13359            met_quorum: true,
13360            uncontested: Some("fixture".into()),
13361        });
13362        runner.state.graft = Some(GraftRecord {
13363            pre: pre.clone(),
13364            post: None,
13365            lines: None,
13366            limit: None,
13367            agent: None,
13368            outcome: GraftOutcome::Running,
13369            requested: Vec::new(),
13370            adoption: None,
13371            adoption_error: None,
13372        });
13373        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13374        git::commit_all(&winner.worktree, "unfinished graft")
13375            .await
13376            .unwrap();
13377        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13378        let leftover = winner.worktree.join("leftover");
13379        std::fs::write(&leftover, "untrusted edits").unwrap();
13380        let path = runner.state.repo.join(".git/updater.lock");
13381        let external = std::fs::File::create(&path).unwrap();
13382        assert_eq!(
13383            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13384            0
13385        );
13386        assert!(
13387            runner
13388                .graft()
13389                .await
13390                .unwrap_err()
13391                .is::<crate::repo_lock::ResourceBlocked>()
13392        );
13393        assert!(matches!(
13394            runner.state.graft.as_ref().unwrap().outcome,
13395            GraftOutcome::Running
13396        ));
13397        assert_eq!(
13398            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13399            unfinished
13400        );
13401        assert!(leftover.exists());
13402        drop(external);
13403        std::fs::remove_file(&path).unwrap();
13404        std::fs::create_dir(&path).unwrap();
13405        assert!(runner.graft().await.is_err());
13406        assert!(matches!(
13407            runner.state.graft.as_ref().unwrap().outcome,
13408            GraftOutcome::Running
13409        ));
13410        assert_eq!(
13411            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13412            unfinished
13413        );
13414        assert!(leftover.exists());
13415        std::fs::remove_dir(&path).unwrap();
13416        runner.graft().await.unwrap();
13417        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13418        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13419        assert!(!leftover.exists());
13420    }
13421
13422    #[tokio::test]
13423    #[cfg(unix)]
13424    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13425        use std::os::fd::AsRawFd as _;
13426        let tmp = tempfile::tempdir().unwrap();
13427        let mut runner = registered_runner(tmp.path()).await;
13428        runner.state.prep_complete = Some(false);
13429        runner.state.config.graph.timeout_resource_wait = 1;
13430        runner.state.save().unwrap();
13431        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13432        assert_eq!(
13433            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13434            0
13435        );
13436        runner.execute().await.unwrap();
13437        let disk = RunState::load(&runner.state.id).unwrap();
13438        assert!(disk.driver_exited);
13439        assert_eq!(disk.driver_pid, None);
13440        assert_eq!(runner.state.status, RunStatus::Blocked);
13441        assert_eq!(
13442            runner.state.resource_block.as_ref().unwrap().kind,
13443            "repository coordination"
13444        );
13445        assert!(runner.state.prep_failure.is_none());
13446        assert!(runner.state.seats.is_empty());
13447        runner.pause.park();
13448        // Include headroom for Git subprocesses under parallel test load;
13449        // the repository contention deadline itself remains one second.
13450        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13451            .await
13452            .unwrap()
13453            .unwrap();
13454        assert!(runner.state.parked);
13455        assert!(runner.state.prep_failure.is_none());
13456        let disk = RunState::load(&runner.state.id).unwrap();
13457        assert!(disk.driver_exited);
13458        assert_eq!(disk.driver_pid, None);
13459    }
13460
13461    #[tokio::test]
13462    #[cfg(unix)]
13463    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13464        use std::os::fd::AsRawFd as _;
13465        let tmp = tempfile::tempdir().unwrap();
13466        let mut runner = registered_runner(tmp.path()).await;
13467        runner.state.prep_complete = Some(false);
13468        runner.state.config.graph.timeout_resource_wait = 30;
13469        runner.state.save().unwrap();
13470        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13471        assert_eq!(
13472            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13473            0
13474        );
13475        assert!(
13476            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13477                .await
13478                .is_err()
13479        );
13480        let disk = RunState::load(&runner.state.id).unwrap();
13481        assert!(disk.driver_exited);
13482        assert_eq!(disk.driver_pid, None);
13483        assert!(disk.status.resumable());
13484        assert!(runner.state.driver_exited);
13485        assert_eq!(runner.state.driver_pid, None);
13486        runner.state.save().unwrap();
13487        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13488    }
13489
13490    #[tokio::test]
13491    #[cfg(unix)]
13492    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13493        let tmp = tempfile::tempdir().unwrap();
13494        let mut runner = registered_runner(tmp.path()).await;
13495        let winner = runner.state.candidates[0].clone();
13496        let cache = tmp.path().join("cache");
13497        let started = tmp.path().join("pre-gate-started");
13498        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13499        runner.state.config.verify.gate =
13500            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13501        let home = crate::run::home();
13502        let owner =
13503            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13504        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13505            crate::cache::AcquireOutcome::Acquired(g) => g,
13506            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13507        };
13508        let mut gate = Box::pin(runner.run_gate(&winner));
13509        assert!(
13510            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13511                .await
13512                .is_err()
13513        );
13514        assert!(!started.exists(), "pre-gate must wait for the cache too");
13515        // Simulate the wave that already owns the cache entering invoke.
13516        // It must obtain the editor reservation and finish without waiting
13517        // for the gate that is queued behind its cache.
13518        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13519        let editor = crate::repo_lock::SeatLease::checkout_with(
13520            &winner.worktree,
13521            true,
13522            crate::config::test_coordination().as_ref(),
13523            &wait,
13524        )
13525        .await
13526        .unwrap();
13527        drop(editor);
13528        guard.release();
13529        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13530            .await
13531            .unwrap()
13532            .unwrap();
13533        drop(gate);
13534        assert!(outcomes.iter().all(CommandOutcome::ok));
13535        assert!(started.exists());
13536    }
13537
13538    #[tokio::test]
13539    #[cfg(unix)]
13540    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13541        use std::os::fd::AsRawFd as _;
13542
13543        ask_test_home();
13544        let tmp = tempfile::tempdir().unwrap();
13545        let repo = tmp.path().join("repo");
13546        std::fs::create_dir_all(&repo).unwrap();
13547        init_repo(&repo);
13548        let common = repo.join(".git");
13549        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13550        let mut runner = runner_at(RunStatus::Prep);
13551        runner.state.repo = repo.clone();
13552        runner.state.config.coordination = crate::config::test_coordination();
13553        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13554        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13555        runner.state.config.graph.advise = false;
13556        runner.roles.implementers = vec![spec("implement")];
13557        runner.roles.judges = vec![spec("judge")];
13558        runner.prep().await.unwrap();
13559        let winner = runner.state.candidates[0].clone();
13560        // Emulate edits already left by a completed seat. The fixture CLI
13561        // only reports completion; it performs no repository operations.
13562        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13563        let completed = tmp.path().join("seat-completed");
13564        runner.roles.implementers[0].command = vec![
13565            "sh".into(),
13566            "-c".into(),
13567            format!("touch '{}'; echo completed", completed.display()),
13568        ];
13569        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13570        assert_eq!(
13571            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13572            0
13573        );
13574        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13575        let mut implement = Box::pin(runner.implement());
13576        tokio::select! {
13577            result = &mut implement => panic!("rescue must wait: {result:?}"),
13578            result = tokio::time::timeout(Duration::from_secs(10), async {
13579                while !completed.exists() {
13580                    tokio::time::sleep(Duration::from_millis(20)).await;
13581                }
13582            }) => result.unwrap(),
13583        }
13584        assert!(
13585            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13586                .await
13587                .is_err()
13588        );
13589        assert_eq!(
13590            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13591            before
13592        );
13593        assert_eq!(
13594            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13595            0
13596        );
13597        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13598            .await
13599            .unwrap()
13600            .unwrap();
13601        drop(implement);
13602        assert_ne!(
13603            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13604            before
13605        );
13606        assert!(git::is_clean(&winner.worktree).await.unwrap());
13607
13608        let started = tmp.path().join("pre-gate-started");
13609        let proceed = tmp.path().join("pre-gate-proceed");
13610        let checked = tmp.path().join("gate-ran");
13611        runner.state.config.verify.pre_gate = vec![format!(
13612            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13613            started.display(),
13614            proceed.display()
13615        )];
13616        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13617        assert_eq!(
13618            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13619            0
13620        );
13621        let mut gate = Box::pin(runner.run_gate(&winner));
13622        assert!(
13623            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13624                .await
13625                .is_err()
13626        );
13627        assert!(!started.exists());
13628        assert!(!checked.exists());
13629        assert!(git::is_clean(&winner.worktree).await.unwrap());
13630        assert_eq!(
13631            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13632            0
13633        );
13634        tokio::select! {
13635            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13636            result = tokio::time::timeout(Duration::from_secs(10), async {
13637                while !started.exists() {
13638                    tokio::time::sleep(Duration::from_millis(20)).await;
13639                }
13640            }) => result.unwrap(),
13641        }
13642        assert_eq!(
13643            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13644            -1,
13645            "mechanical edits must hold the real lock"
13646        );
13647        std::fs::write(proceed, "ready").unwrap();
13648        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13649            .await
13650            .unwrap()
13651            .unwrap();
13652        drop(gate);
13653        assert!(outcomes.iter().all(CommandOutcome::ok));
13654        assert!(checked.exists());
13655        assert!(runner.state.pre_gate_commit.is_some());
13656        assert!(git::is_clean(&winner.worktree).await.unwrap());
13657        assert_eq!(
13658            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13659            0,
13660            "gate completion must release coordination"
13661        );
13662        drop(outsider);
13663        std::fs::remove_file(common.join("updater.lock")).unwrap();
13664        std::fs::create_dir(common.join("updater.lock")).unwrap();
13665        std::fs::remove_file(&started).unwrap();
13666        std::fs::remove_file(&checked).unwrap();
13667        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13668        let error = runner.run_gate(&winner).await.unwrap_err();
13669        assert!(
13670            error
13671                .to_string()
13672                .contains("acquire mandatory repository lock")
13673        );
13674        assert!(
13675            !started.exists(),
13676            "failed acquisition must prevent pre-gate"
13677        );
13678        assert!(!checked.exists(), "failed acquisition must prevent gate");
13679        assert!(git::is_clean(&winner.worktree).await.unwrap());
13680        assert_eq!(
13681            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13682            head
13683        );
13684    }
13685
13686    #[tokio::test]
13687    #[cfg(unix)]
13688    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13689        use std::os::fd::AsRawFd as _;
13690        use std::time::Duration;
13691
13692        ask_test_home();
13693        let tmp = tempfile::tempdir().unwrap();
13694        let repo = tmp.path().join("repo");
13695        std::fs::create_dir_all(&repo).unwrap();
13696        init_repo(&repo);
13697        let common = repo.join(".git");
13698        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13699        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13700        assert_eq!(
13701            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13702            0
13703        );
13704        let mut runner = runner_at(RunStatus::Prep);
13705        runner.state.repo = repo.clone();
13706        runner.state.config.coordination = crate::config::test_coordination();
13707        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13708        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13709        runner.state.config.graph.advise = true;
13710        runner.state.config.graph.advisors = 1;
13711        runner.roles.implementers = vec![spec("implement")];
13712        runner.roles.judges = vec![spec("judge")];
13713        let root = runner.state.worktree_root();
13714        let started = tmp.path().join("setup-started");
13715        let proceed = tmp.path().join("setup-proceed");
13716        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13717            run: Some(format!(
13718                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13719                started.display(),
13720                proceed.display()
13721            )),
13722            ..Default::default()
13723        }];
13724        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13725        let config = git::git(&repo, &["config", "--local", "--list"])
13726            .await
13727            .unwrap();
13728        let mut prep = Box::pin(runner.prep());
13729        assert!(
13730            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13731                .await
13732                .is_err()
13733        );
13734        assert!(
13735            !root.exists(),
13736            "contention must prevent even checkout creation"
13737        );
13738        assert!(!started.exists(), "contention must prevent setup");
13739        assert_eq!(
13740            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13741            branches
13742        );
13743        assert_eq!(
13744            git::git(&repo, &["config", "--local", "--list"])
13745                .await
13746                .unwrap(),
13747            config
13748        );
13749        drop(lock);
13750        tokio::select! {
13751            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13752            result = tokio::time::timeout(Duration::from_secs(10), async {
13753                while !started.exists() {
13754                    tokio::time::sleep(Duration::from_millis(20)).await;
13755                }
13756            }) => result.unwrap(),
13757        }
13758        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13759        assert_eq!(
13760            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13761            -1
13762        );
13763        assert!(
13764            tokio::time::timeout(
13765                Duration::from_millis(100),
13766                crate::repo_lock::SeatLease::for_checkout(
13767                    &repo,
13768                    true,
13769                    crate::config::test_coordination().as_ref()
13770                )
13771            )
13772            .await
13773            .is_err(),
13774            "preparation must also exclude local editors"
13775        );
13776        std::fs::write(proceed, "ready").unwrap();
13777        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13778            .await
13779            .unwrap()
13780            .unwrap();
13781        drop(prep);
13782        assert_eq!(runner.state.prep_complete, Some(true));
13783        assert!(root.join("judge-1").exists());
13784        assert!(root.join("advisor-1").exists());
13785        assert_eq!(
13786            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13787            0
13788        );
13789    }
13790
13791    #[tokio::test]
13792    #[cfg(unix)]
13793    async fn preparation_lock_failure_prevents_repository_changes() {
13794        ask_test_home();
13795        let tmp = tempfile::tempdir().unwrap();
13796        let repo = tmp.path().join("repo");
13797        std::fs::create_dir_all(&repo).unwrap();
13798        init_repo(&repo);
13799        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13800        // A directory at the required lock path makes opening the real lock
13801        // fail, rather than merely exercising an invalid registration.
13802        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13803        let mut runner = runner_at(RunStatus::Prep);
13804        runner.state.repo = repo.clone();
13805        runner.state.config.coordination = crate::config::test_coordination();
13806        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13807        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13808        runner.roles.implementers = vec![spec("implement")];
13809        let error = runner.prep().await.unwrap_err();
13810        assert!(
13811            error
13812                .to_string()
13813                .contains("acquire mandatory repository lock")
13814        );
13815        assert!(!runner.state.worktree_root().exists());
13816        assert!(runner.state.candidates.is_empty());
13817        assert!(!repo.join(".git/config.worktree").exists());
13818        assert_eq!(
13819            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13820            "* main"
13821        );
13822    }
13823
13824    #[tokio::test]
13825    async fn preparation_records_creation_intent_before_git_can_fail() {
13826        ask_test_home();
13827        let tmp = tempfile::tempdir().unwrap();
13828        let repo = tmp.path().join("repo");
13829        std::fs::create_dir_all(&repo).unwrap();
13830        init_repo(&repo);
13831        let mut runner = runner_at(RunStatus::Prep);
13832        runner.state.repo = repo.clone();
13833        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13834        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13835        runner.state.config.blind.commit_msg_hook = false;
13836        runner.roles.implementers = vec![spec("one")];
13837        let label = blind::assign_labels(1, runner.state.seed)[0];
13838        let branch = runner.state.branch_for(label);
13839        // A conflicting branch makes Git refuse creation before a candidate
13840        // can be registered. The durable intent must still identify it.
13841        git::git(&repo, &["branch", &branch]).await.unwrap();
13842        assert!(runner.prep().await.is_err());
13843        let saved = RunState::load(&runner.state.id).unwrap();
13844        assert!(saved.candidates.is_empty());
13845        let event = saved
13846            .events
13847            .iter()
13848            .find(|e| e.node == "prep-intent")
13849            .unwrap();
13850        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13851        assert_eq!(intent["branch"], branch);
13852        assert_eq!(intent["start"], runner.state.base_commit);
13853        assert_eq!(
13854            intent["worktree"],
13855            runner
13856                .state
13857                .worktree_root()
13858                .join(format!("cand-{label}"))
13859                .to_string_lossy()
13860                .as_ref()
13861        );
13862        assert_eq!(intent["picks"], serde_json::json!([]));
13863    }
13864
13865    #[tokio::test]
13866    async fn invalid_pinned_sources_create_no_partial_worktrees() {
13867        ask_test_home();
13868        let tmp = tempfile::tempdir().unwrap();
13869        let repo = tmp.path().join("repo");
13870        std::fs::create_dir_all(&repo).unwrap();
13871        init_repo(&repo);
13872        let mut runner = runner_at(RunStatus::Prep);
13873        runner.state.repo = repo.clone();
13874        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13875        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13876        runner.state.sources.pick_commits.push(refs::Input {
13877            token: "missing".into(),
13878            sha: "0".repeat(40),
13879            provenance: refs::Provenance::ExplicitCli,
13880        });
13881        assert!(runner.prep().await.is_err());
13882        assert!(runner.state.candidates.is_empty());
13883        assert!(!runner.state.worktree_root().exists());
13884    }
13885
13886    #[tokio::test]
13887    async fn prose_and_legacy_inference_never_import_into_candidates() {
13888        ask_test_home();
13889        let tmp = tempfile::tempdir().unwrap();
13890        let repo = tmp.path().join("repo");
13891        std::fs::create_dir_all(&repo).unwrap();
13892        init_repo(&repo);
13893        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13894        git::git(&repo, &["checkout", "-b", "magi/history/A"])
13895            .await
13896            .unwrap();
13897        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13898        git::git(&repo, &["add", "."]).await.unwrap();
13899        git::git(&repo, &["commit", "-m", "historical implementation"])
13900            .await
13901            .unwrap();
13902        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13903        git::git(&repo, &["checkout", "main"]).await.unwrap();
13904        for instruction in [
13905            format!("Example: magi/history/A at {source}"),
13906            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13907            format!("Never import magi/history/A or {source}"),
13908            format!("Conversation answer: I mentioned {source} only as background"),
13909        ] {
13910            let mut runner = runner_at(RunStatus::Prep);
13911            runner.state.repo = repo.clone();
13912            runner.state.base_commit = base.clone();
13913            runner.state.instruction = instruction.clone();
13914            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13915            runner.state.config.blind.commit_msg_hook = false;
13916            runner.state.seeds = vec![refs::Seed {
13917                token: "magi/history/A".into(),
13918                sha: source.clone(),
13919                kind: refs::SeedKind::Unmerged,
13920                branch: true,
13921                detail: String::new(),
13922            }];
13923            runner.roles.implementers = vec![spec("one")];
13924            assert_eq!(seeded_instruction(&runner.state), instruction);
13925            // Model an old record with inferred seeds but no structured input field.
13926            let mut legacy = serde_json::to_value(&runner.state).unwrap();
13927            legacy.as_object_mut().unwrap().remove("sources");
13928            runner.state = serde_json::from_value(legacy).unwrap();
13929            assert_eq!(runner.state.sources, refs::Sources::default());
13930            runner.state.save().unwrap();
13931            runner.state = RunState::load(&runner.state.id).unwrap();
13932            runner.prep().await.unwrap();
13933            for candidate in &runner.state.candidates {
13934                assert_eq!(
13935                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13936                    base
13937                );
13938                assert!(!candidate.worktree.join("unwanted").exists());
13939            }
13940        }
13941    }
13942
13943    #[tokio::test]
13944    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13945        ask_test_home();
13946        let tmp = tempfile::tempdir().unwrap();
13947        let repo = tmp.path().join("repo");
13948        std::fs::create_dir_all(&repo).unwrap();
13949        init_repo(&repo);
13950        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13951        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13952        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13953        git::git(&repo, &["add", "."]).await.unwrap();
13954        git::git(&repo, &["commit", "-m", "required seed"])
13955            .await
13956            .unwrap();
13957        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13958        git::git(&repo, &["checkout", "main"]).await.unwrap();
13959        let mut runner = runner_at(RunStatus::Prep);
13960        runner.state.repo = repo.clone();
13961        runner.state.base_commit = base.clone();
13962        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13963        runner.state.config.graph.advise = false;
13964        runner.state.config.blind.commit_msg_hook = false;
13965        runner.state.sources =
13966            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13967                .await
13968                .unwrap();
13969        runner.roles.implementers = vec![spec("one"), spec("two")];
13970        runner.roles.judges = vec![spec("judge")];
13971        let label = blind::assign_labels(2, runner.state.seed)[0];
13972        let branch = runner.state.branch_for(label);
13973        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13974        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13975            .await
13976            .unwrap();
13977        runner.state.candidates.push(Candidate {
13978            index: 0,
13979            label,
13980            agent: "one".into(),
13981            branch,
13982            worktree,
13983            summary: String::new(),
13984            stat: String::new(),
13985            files: 0,
13986            commits: 0,
13987            empty: false,
13988            failed: None,
13989            verified_noop: None,
13990            duration_ms: 0,
13991            folded: false,
13992        });
13993        runner.state.save().unwrap();
13994        // Reload the checkpoint immediately before its first cherry-pick.
13995        runner.state = RunState::load(&runner.state.id).unwrap();
13996        assert_eq!(runner.state.prep_complete, Some(false));
13997        runner.prep().await.unwrap();
13998        assert_eq!(runner.state.prep_complete, Some(true));
13999        assert_eq!(runner.state.candidates.len(), 2);
14000        for candidate in &runner.state.candidates {
14001            assert_eq!(
14002                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14003                git::tree_of(&repo, &seed).await.unwrap()
14004            );
14005            assert_eq!(
14006                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14007                "required commit"
14008            );
14009        }
14010        assert!(runner.state.worktree_root().join("judge-1").exists());
14011        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14012        runner.prep().await.unwrap();
14013        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14014
14015        runner.state.prep_complete = Some(false);
14016        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14017            run: Some(format!(
14018                "test -f '{}'",
14019                tmp.path().join("dependency-ready").display()
14020            )),
14021            ..Default::default()
14022        }];
14023        assert!(runner.prep().await.is_err());
14024        assert!(runner.state.prep_failure.is_some());
14025        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14026        runner.state = RunState::load(&runner.state.id).unwrap();
14027        assert!(
14028            runner.prep().await.is_err(),
14029            "correcting the cause alone must not auto-retry"
14030        );
14031        let failed_at = runner.state.events.last().unwrap().at;
14032        assert!(
14033            runner.retry_preparation(failed_at).is_err(),
14034            "old approvals cannot unlock the hold"
14035        );
14036        let mut task = crate::queue::Task::new(
14037            "retry".into(),
14038            "retry setup".into(),
14039            repo.clone(),
14040            crate::queue::Source::Human,
14041        );
14042        task.start(runner.state.id.clone());
14043        task.hold_machine(Some("setup failed".into()));
14044        task.release_operator("cli", None);
14045        runner
14046            .retry_preparation(task.resumption.as_ref().unwrap().at)
14047            .unwrap();
14048        runner.prep().await.unwrap();
14049        assert!(runner.state.prep_failure.is_none());
14050        assert_eq!(runner.state.prep_complete, Some(true));
14051        assert_eq!(runner.state.candidates.len(), 2);
14052        assert!(runner.state.worktree_root().join("judge-1").exists());
14053        for candidate in &runner.state.candidates {
14054            assert_eq!(
14055                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14056                git::tree_of(&repo, &seed).await.unwrap()
14057            );
14058        }
14059    }
14060
14061    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14062        let mut heads = Vec::new();
14063        for candidate in candidates {
14064            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14065        }
14066        heads
14067    }
14068
14069    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14070        ask_test_home();
14071        let tmp = tempfile::tempdir().unwrap();
14072        let repo = tmp.path().join("repo");
14073        std::fs::create_dir(&repo).unwrap();
14074        init_repo(&repo);
14075        let mut runner = runner_at(RunStatus::Reviewing);
14076        runner.state.repo = repo.clone();
14077        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14078        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14079        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14080            let branch = format!("candidate-{label}");
14081            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14082            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14083                .await
14084                .unwrap();
14085            if label == 'B' {
14086                std::fs::write(worktree.join("change"), "winner").unwrap();
14087                git::rescue_commit(&worktree, "candidate change")
14088                    .await
14089                    .unwrap();
14090            }
14091            runner.state.candidates.push(Candidate {
14092                index,
14093                label,
14094                agent: "alpha".into(),
14095                branch,
14096                worktree,
14097                summary: "completed reply".into(),
14098                stat: String::new(),
14099                files: usize::from(label == 'B'),
14100                commits: usize::from(label == 'B'),
14101                empty: label == 'A',
14102                failed: None,
14103                verified_noop: None,
14104                duration_ms: 1,
14105                folded: false,
14106            });
14107        }
14108        runner.state.judge_skipped = true;
14109        runner.state.tally = Some(Tally {
14110            first_choice: BTreeMap::new(),
14111            borda: BTreeMap::new(),
14112            winner: 'B',
14113            rankings: 0,
14114            unanimous_initial: false,
14115            deliberated: false,
14116            changed_votes: 0,
14117            unanimous_final: false,
14118            tie_break: None,
14119            judges: 0,
14120            present: 0,
14121            quorum: 0,
14122            met_quorum: true,
14123            uncontested: Some("only B produced a change".into()),
14124        });
14125        runner.state.save().unwrap();
14126        (tmp, runner)
14127    }
14128
14129    #[tokio::test]
14130    #[cfg(unix)]
14131    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14132        for (solo, failed_checkpoint) in
14133            [(true, false), (false, false), (true, true), (false, true)]
14134        {
14135            let (_tmp, mut runner) = preservation_resume_fixture().await;
14136            if solo {
14137                runner.state.candidates.retain(|c| c.label == 'A');
14138            }
14139            runner.state.candidates[0].empty = false;
14140            runner.state.candidates[0].summary.clear();
14141            runner.state.tally = None;
14142            runner.state.judge_skipped = false;
14143            runner.state.status = RunStatus::Implementing;
14144            runner.state.config.disk.min_free_bytes = u64::MAX;
14145            let mut implementer = spec("alpha");
14146            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14147            runner.roles.implementers = vec![implementer];
14148            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14149            if failed_checkpoint {
14150                std::fs::create_dir_all(&patch).unwrap();
14151            }
14152            runner.state.save().unwrap();
14153            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14154            runner.state = RunState::load(&runner.state.id).unwrap();
14155            assert_eq!(runner.state.status, RunStatus::Blocked);
14156            assert!(runner.state.interrupted_implementations.contains(&'A'));
14157            assert_eq!(runner.state.candidates[0].commits, 0);
14158            assert!(!runner.state.candidates[0].empty);
14159            assert!(runner.state.candidates[0].failed.is_none());
14160            if failed_checkpoint {
14161                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14162                std::fs::remove_dir(patch).unwrap();
14163            }
14164            runner.state.resource_block = None;
14165            runner.state.config.disk.min_free_bytes = 0;
14166            runner.roles.implementers[0].command = vec![
14167                "sh".into(),
14168                "-c".into(),
14169                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14170            ];
14171            runner.implement().await.unwrap();
14172            assert!(runner.state.interrupted_implementations.is_empty());
14173            assert!(runner.state.candidate_preservation.is_empty());
14174            assert_eq!(runner.state.candidates[0].commits, 1);
14175            assert!(!runner.state.candidates[0].empty);
14176            assert!(
14177                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14178                    .unwrap()
14179                    .contains("retried empty interrupted seat")
14180            );
14181            if !solo {
14182                assert_eq!(runner.state.candidates[1].commits, 1);
14183            }
14184        }
14185    }
14186
14187    #[tokio::test]
14188    #[cfg(unix)]
14189    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14190        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14191            let (_tmp, mut runner) = preservation_resume_fixture().await;
14192            runner.state.candidates.retain(|c| c.label == 'A');
14193            runner.state.candidates[0].empty = false;
14194            runner.state.candidates[0].summary.clear();
14195            runner.state.tally = None;
14196            runner.state.judge_skipped = false;
14197            runner.state.status = RunStatus::Implementing;
14198            // Force the real post-spawn watchdog, independently of available
14199            // disk space. Dirty/committed work predates its first measurement.
14200            runner.state.config.disk.min_free_bytes = u64::MAX;
14201            let wt = runner.state.candidates[0].worktree.clone();
14202            std::fs::write(
14203                wt.join("interrupted-source"),
14204                "retain implementation work\n",
14205            )
14206            .unwrap();
14207            if committed {
14208                git::rescue_commit(&wt, "implementation checkpoint")
14209                    .await
14210                    .unwrap();
14211            }
14212            let lock = git::git(
14213                &wt,
14214                &[
14215                    "rev-parse",
14216                    "--path-format=absolute",
14217                    "--git-path",
14218                    "index.lock",
14219                ],
14220            )
14221            .await
14222            .unwrap();
14223            if block_rescue {
14224                std::fs::write(&lock, "fixture-owned lock").unwrap();
14225            }
14226            let mut implementer = spec("alpha");
14227            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14228            runner.roles.implementers = vec![implementer];
14229            runner.state.save().unwrap();
14230            let result = runner.implement().await;
14231            assert_eq!(result.is_err(), block_rescue);
14232            assert_eq!(
14233                runner.state.resource_block.as_ref().unwrap().kind,
14234                "disk pressure"
14235            );
14236            assert!(
14237                runner
14238                    .state
14239                    .events
14240                    .iter()
14241                    .any(|e| e.message.contains("agent execution interrupted"))
14242            );
14243            assert!(
14244                runner
14245                    .state
14246                    .dir()
14247                    .join("artifacts/candidate-A-recovery/staged.patch")
14248                    .exists()
14249            );
14250            if block_rescue {
14251                let saved = RunState::load(&runner.state.id).unwrap();
14252                assert!(saved.candidate_preservation.contains_key(&'A'));
14253                assert!(!saved.candidates[0].empty);
14254                assert_eq!(
14255                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14256                    "retain implementation work\n"
14257                );
14258                std::fs::remove_file(lock).unwrap();
14259                runner.state = saved;
14260            } else {
14261                assert_eq!(runner.state.status, RunStatus::Blocked);
14262                assert_eq!(runner.state.candidates[0].commits, 1);
14263            }
14264            // Recovery must save the candidate before deciding whether another
14265            // implementation is needed. No implementer is available on retry.
14266            runner.roles.implementers.clear();
14267            runner.state.config.disk.min_free_bytes = 0;
14268            runner.state.resource_block = None;
14269            runner.implement().await.unwrap();
14270            assert!(runner.state.candidate_preservation.is_empty());
14271            assert_eq!(runner.state.candidates[0].commits, 1);
14272            assert!(!runner.state.candidates[0].empty);
14273            assert!(git::is_clean(&wt).await.unwrap());
14274            assert!(
14275                crate::run::read_artifact(&runner.state, "cand-A.patch")
14276                    .unwrap()
14277                    .contains("retain implementation work")
14278            );
14279        }
14280    }
14281
14282    #[tokio::test]
14283    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14284        let (_tmp, mut runner) = preservation_resume_fixture().await;
14285        // Schema 15 already confirmed the unfolded no-change result.
14286        runner.implement().await.unwrap();
14287        assert!(runner.state.candidate_preservation.is_empty());
14288        assert!(
14289            !runner
14290                .state
14291                .dir()
14292                .join("artifacts/candidate-A-recovery")
14293                .exists()
14294        );
14295        runner.fold_losers().await.unwrap();
14296        assert!(!runner.state.candidates[0].worktree.exists());
14297        for schema in [14, crate::run::SCHEMA] {
14298            runner.state.schema = schema;
14299            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14300            runner.state.save().unwrap();
14301            runner.state = RunState::load(&runner.state.id).unwrap();
14302            runner.implement().await.unwrap();
14303            assert!(runner.state.candidate_preservation.is_empty());
14304            assert!(runner.state.candidates[0].folded);
14305            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14306        }
14307    }
14308
14309    #[tokio::test]
14310    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14311        let (_tmp, mut runner) = preservation_resume_fixture().await;
14312        let wt = runner.state.candidates[0].worktree.clone();
14313        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14314        runner.state.schema = 14;
14315        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14316        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14317        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14318        for _ in 0..2 {
14319            assert!(runner.implement().await.is_err());
14320            assert_eq!(runner.state.status, RunStatus::Blocked);
14321            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14322            assert!(
14323                runner.state.candidates[0].empty,
14324                "old verdict must remain unusable"
14325            );
14326            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14327            assert!(runner.state.judge_skipped);
14328            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14329            assert!(
14330                crate::run::read_artifact(&runner.state, "cand-A.patch")
14331                    .unwrap()
14332                    .contains("owned edits")
14333            );
14334            assert!(wt.exists());
14335            assert_eq!(
14336                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14337                    .await
14338                    .unwrap(),
14339                1
14340            );
14341            runner.state = RunState::load(&runner.state.id).unwrap();
14342        }
14343    }
14344
14345    #[tokio::test]
14346    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14347        let (_tmp, mut runner) = preservation_resume_fixture().await;
14348        let wt = runner.state.candidates[0].worktree.clone();
14349        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14350        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14351        let index_lock = PathBuf::from(
14352            git::git(
14353                &wt,
14354                &[
14355                    "rev-parse",
14356                    "--path-format=absolute",
14357                    "--git-path",
14358                    "index.lock",
14359                ],
14360            )
14361            .await
14362            .unwrap(),
14363        );
14364        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14365        runner.state.schema = 14;
14366        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14367        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14368        assert!(runner.implement().await.is_err());
14369        assert!(runner.after_implement().is_err());
14370        assert!(runner.fold_losers().await.is_err());
14371        assert_eq!(
14372            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14373            "failed seat edits"
14374        );
14375        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14376        runner.state = RunState::load(&runner.state.id).unwrap();
14377        std::fs::remove_file(index_lock).unwrap();
14378        // Recovery finds changes that the old uncontested result never judged.
14379        assert!(runner.implement().await.is_err());
14380        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14381        assert!(runner.state.candidates[0].empty);
14382        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14383        assert!(
14384            crate::run::read_artifact(&runner.state, "cand-A.patch")
14385                .unwrap()
14386                .contains("failed seat edits")
14387        );
14388        assert!(wt.exists());
14389    }
14390
14391    #[tokio::test]
14392    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14393        let (_tmp, mut runner) = preservation_resume_fixture().await;
14394        let repo = runner.state.repo.clone();
14395        let common = git::git(
14396            &repo,
14397            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14398        )
14399        .await
14400        .unwrap();
14401        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14402        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14403        assert!(runner.fold_losers().await.is_err());
14404        assert!(!runner.state.candidates[0].worktree.exists());
14405        assert!(!runner.state.candidates[0].folded);
14406        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14407        runner.state = RunState::load(&runner.state.id).unwrap();
14408        std::fs::remove_file(branch_lock).unwrap();
14409        let judge = runner.state.worktree_root().join("judge-1");
14410        git::worktree_add_detached(&repo, &judge, "main")
14411            .await
14412            .unwrap();
14413        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14414        runner.roles.judges = vec![spec("judge")];
14415        assert!(runner.fold_losers().await.is_err());
14416        runner.state = RunState::load(&runner.state.id).unwrap();
14417        assert!(
14418            runner.state.candidates[0].folded,
14419            "candidate progress saved before judge failure"
14420        );
14421        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14422        assert!(judge.join("uncommitted").exists());
14423        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14424        runner.fold_losers().await.unwrap();
14425        assert!(!judge.exists());
14426        runner.fold_losers().await.unwrap();
14427    }
14428
14429    #[tokio::test]
14430    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14431        let (_tmp, mut runner) = preservation_resume_fixture().await;
14432        runner.state.candidates.retain(|c| c.label == 'B');
14433        runner.state.tally = None;
14434        runner.state.judge_skipped = false;
14435        runner.state.prep_complete = Some(true);
14436        runner.state.config.graph.advise = false;
14437        runner.state.config.graph.selected_implementer = Some("alpha".into());
14438        let wt = runner.state.candidates[0].worktree.clone();
14439        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14440        runner
14441            .state
14442            .candidate_preservation
14443            .insert('B', "awaiting persistence".into());
14444        let index_lock = git::git(
14445            &wt,
14446            &[
14447                "rev-parse",
14448                "--path-format=absolute",
14449                "--git-path",
14450                "index.lock",
14451            ],
14452        )
14453        .await
14454        .unwrap();
14455        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14456        runner.state.save().unwrap();
14457        assert!(runner.execute_graph().await.is_err());
14458        assert_eq!(runner.state.status, RunStatus::Blocked);
14459        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14460        std::fs::remove_file(index_lock).unwrap();
14461        // Park at the next graph boundary: this tests the real selected-seat
14462        // hold guard without dispatching reviewers or publishing anything.
14463        for _ in 0..2 {
14464            runner.state = RunState::load(&runner.state.id).unwrap();
14465            runner.pause.park();
14466            runner.execute_graph().await.unwrap();
14467            assert_eq!(runner.state.status, RunStatus::Implementing);
14468            assert!(runner.state.candidate_preservation.is_empty());
14469            assert_eq!(runner.state.candidates[0].commits, 2);
14470            assert!(runner.state.tally.is_none());
14471        }
14472        runner.implement().await.unwrap();
14473        assert_eq!(runner.state.status, RunStatus::Judging);
14474        // A genuine selected-implementer hold must still require release.
14475        runner.state.status = RunStatus::Blocked;
14476        runner.state.candidates[0].failed = Some("implementation failed".into());
14477        runner.state.save().unwrap();
14478        let error = runner.execute_graph().await.unwrap_err();
14479        assert!(
14480            error
14481                .to_string()
14482                .contains("selected implementation is held")
14483        );
14484    }
14485
14486    #[tokio::test]
14487    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14488        let (_tmp, mut runner) = preservation_resume_fixture().await;
14489        let repo = runner.state.repo.clone();
14490        let candidate = runner.state.candidates[0].clone();
14491        // Remove only the disposable fixture directory, leaving Git's linked
14492        // worktree registration and branch intact, as in a vanished checkout.
14493        let registered = git::worktree_holding(&repo, &candidate.branch)
14494            .await
14495            .unwrap()
14496            .unwrap();
14497        assert_eq!(
14498            registered.canonicalize().unwrap(),
14499            candidate.worktree.canonicalize().unwrap()
14500        );
14501        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14502        assert_eq!(
14503            git::worktree_holding(&repo, &candidate.branch)
14504                .await
14505                .unwrap(),
14506            Some(registered)
14507        );
14508        assert!(
14509            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14510            "the stale registration initially keeps the branch checked out"
14511        );
14512        runner.fold_losers().await.unwrap();
14513        assert!(runner.state.candidates[0].folded);
14514        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14515        assert_eq!(
14516            git::worktree_holding(&repo, &candidate.branch)
14517                .await
14518                .unwrap(),
14519            None
14520        );
14521        runner.state = RunState::load(&runner.state.id).unwrap();
14522        runner.fold_losers().await.unwrap();
14523    }
14524
14525    #[tokio::test]
14526    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14527        let (_tmp, mut runner) = preservation_resume_fixture().await;
14528        let repo = runner.state.repo.clone();
14529        let candidate = runner.state.candidates[0].clone();
14530        let winner = runner.state.candidates[1].clone();
14531        let registered = git::worktree_holding(&repo, &candidate.branch)
14532            .await
14533            .unwrap()
14534            .unwrap();
14535        assert_eq!(
14536            registered.canonicalize().unwrap(),
14537            candidate.worktree.canonicalize().unwrap()
14538        );
14539        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14540        assert_eq!(
14541            git::worktree_holding(&repo, &candidate.branch)
14542                .await
14543                .unwrap(),
14544            Some(registered)
14545        );
14546        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14547
14548        for _ in 0..2 {
14549            fold_run(&mut runner.state, false, &crate::run::home())
14550                .await
14551                .unwrap();
14552            runner.state = RunState::load(&runner.state.id).unwrap();
14553            assert!(runner.state.candidates[0].folded);
14554            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14555            assert_eq!(
14556                git::worktree_holding(&repo, &candidate.branch)
14557                    .await
14558                    .unwrap(),
14559                None
14560            );
14561            assert!(!runner.state.candidates[1].folded);
14562            assert!(winner.worktree.exists());
14563            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14564        }
14565    }
14566
14567    #[tokio::test]
14568    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14569        ask_test_home();
14570        let tmp = tempfile::tempdir().unwrap();
14571        let repo = tmp.path().join("repo");
14572        std::fs::create_dir(&repo).unwrap();
14573        init_repo(&repo);
14574        let mut runner = runner_at(RunStatus::Implementing);
14575        runner.state.repo = repo.clone();
14576        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14577        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14578        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14579            let branch = format!("candidate-{label}");
14580            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14581            git::worktree_add_branch(&repo, &wt, &branch, "main")
14582                .await
14583                .unwrap();
14584            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14585            git::git(&wt, &["add", "README.md"]).await.unwrap();
14586            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14587            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14588            runner.state.candidates.push(Candidate {
14589                index,
14590                label,
14591                agent: "alpha".to_owned(),
14592                branch,
14593                worktree: wt,
14594                summary: "completed implementation".to_owned(),
14595                stat: String::new(),
14596                files: 0,
14597                commits: 0,
14598                empty: false,
14599                failed: None,
14600                verified_noop: None,
14601                duration_ms: 42,
14602                folded: false,
14603            });
14604            runner
14605                .state
14606                .candidate_preservation
14607                .insert(label, "awaiting persistence".to_owned());
14608        }
14609        let wt = runner.state.candidates[0].worktree.clone();
14610        let index_lock = git::git(
14611            &wt,
14612            &[
14613                "rev-parse",
14614                "--path-format=absolute",
14615                "--git-path",
14616                "index.lock",
14617            ],
14618        )
14619        .await
14620        .unwrap();
14621        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14622        runner.state.save().unwrap();
14623        assert!(runner.persist_candidates().await.is_err());
14624        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14625        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14626        assert_eq!(
14627            runner.state.candidates[1].commits, 1,
14628            "other results are still collected"
14629        );
14630        assert!(
14631            !runner.state.candidates[0].empty,
14632            "failed rescue is not no change"
14633        );
14634        assert_eq!(
14635            runner.state.candidates[0].summary,
14636            "completed implementation"
14637        );
14638        assert!(runner.after_implement().is_err());
14639        let home = tmp.path().join("fold-home");
14640        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14641        assert!(wt.join("new-file").exists());
14642        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14643        assert!(recovery.join("staged.patch").exists());
14644        assert!(recovery.join("unstaged.patch").exists());
14645        assert_eq!(
14646            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14647            "untracked edits\n"
14648        );
14649        runner.state.begin_pass();
14650        runner.state.end_pass_parked(ParkCause::Upgrade {
14651            from: "1.0".into(),
14652            to: Some("1.1".into()),
14653        });
14654        runner.state.save().unwrap();
14655        runner.state = RunState::load(&runner.state.id).unwrap();
14656        assert!(matches!(
14657            runner.state.passes.last().unwrap().end,
14658            Some(crate::run::PassEnd::Parked {
14659                cause: ParkCause::Upgrade { .. }
14660            })
14661        ));
14662        std::fs::remove_file(index_lock).unwrap();
14663        // No implementer is configured: a persistence retry cannot dispatch a seat.
14664        runner.implement().await.unwrap();
14665        assert!(runner.state.candidate_preservation.is_empty());
14666        assert!(
14667            runner
14668                .state
14669                .candidates
14670                .iter()
14671                .all(|c| c.commits == 1 && !c.empty)
14672        );
14673        assert!(git::is_clean(&wt).await.unwrap());
14674        assert!(
14675            crate::run::read_artifact(&runner.state, "cand-A.patch")
14676                .unwrap()
14677                .contains("later tracked edits")
14678        );
14679    }
14680
14681    #[tokio::test]
14682    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14683        ask_test_home();
14684        let tmp = tempfile::tempdir().unwrap();
14685        let repo = tmp.path().join("repo");
14686        std::fs::create_dir(&repo).unwrap();
14687        init_repo(&repo);
14688        let wt = tmp.path().join("candidate");
14689        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14690            .await
14691            .unwrap();
14692        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14693        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14694        assert!(wt.join("uncommitted").exists());
14695        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14696        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14697        let mut runner = runner_at(RunStatus::Implementing);
14698        runner.state.repo = repo.clone();
14699        runner.state.base_commit = "missing-base".to_owned();
14700        runner.state.candidates.push(Candidate {
14701            index: 0,
14702            label: 'A',
14703            agent: "alpha".into(),
14704            branch: "candidate".into(),
14705            worktree: wt.clone(),
14706            summary: "completed".into(),
14707            stat: String::new(),
14708            files: 0,
14709            commits: 0,
14710            empty: false,
14711            failed: None,
14712            verified_noop: None,
14713            duration_ms: 1,
14714            folded: false,
14715        });
14716        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14717        assert!(
14718            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14719                .await
14720                .is_err()
14721        );
14722        assert!(!runner.state.candidates[0].folded);
14723        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14724        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14725        runner
14726            .state
14727            .candidate_preservation
14728            .insert('A', "pending".into());
14729        assert!(runner.persist_candidates().await.is_err());
14730        assert!(!runner.state.candidates[0].empty);
14731        assert!(!runner.state.candidates[0].folded);
14732        assert!(wt.exists());
14733        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14734        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14735        std::fs::create_dir_all(&patch_path).unwrap();
14736        assert!(
14737            runner.persist_candidates().await.is_err(),
14738            "artifact failure stays pending"
14739        );
14740        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14741        assert!(!runner.state.candidates[0].empty);
14742        std::fs::remove_dir(patch_path).unwrap();
14743        runner.persist_candidates().await.unwrap();
14744        assert!(
14745            runner.state.candidates[0].empty,
14746            "only successful inspection confirms no change"
14747        );
14748    }
14749
14750    fn init_repo(dir: &Path) {
14751        let run = |args: &[&str]| {
14752            let out = std::process::Command::new("git")
14753                .args(args)
14754                .current_dir(dir)
14755                .quiet()
14756                .output()
14757                .expect("spawn git");
14758            assert!(
14759                out.status.success(),
14760                "git {args:?} failed: {}",
14761                String::from_utf8_lossy(&out.stderr)
14762            );
14763        };
14764        run(&["init", "-b", "main"]);
14765        run(&["config", "user.name", "magi test"]);
14766        run(&["config", "user.email", "magi@example.com"]);
14767        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14768        run(&["add", "-A"]);
14769        run(&["commit", "-m", "init"]);
14770    }
14771
14772    // `settle_questions` is what closes the ghost the phone showed: a run's
14773    // seat asked something, the run then ended, and nothing was left to
14774    // abandon the question it left `open`. `HOME` is a process-wide
14775    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14776    // first time it runs in the binary — every test below still reaches the
14777    // same directory whichever call won, and each gets its own run id from
14778    // `RunState::new`, so they never collide there.
14779    fn ask_test_home() {
14780        crate::run::pin_test_home();
14781    }
14782
14783    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14784    /// reads nothing else off it.
14785    fn runner_at(status: RunStatus) -> Runner {
14786        let mut state = RunState::new(
14787            PathBuf::from("/nonexistent/repo"),
14788            "main".to_owned(),
14789            "deadbeef".to_owned(),
14790            "task".to_owned(),
14791            Config::default(),
14792        );
14793        // These fixtures share one home and save concurrently. Short clock
14794        // IDs can collide, making an ownership test read another test's run.
14795        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14796        state.id = format!(
14797            "20000101-000000-{:04x}",
14798            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14799        );
14800        state.status = status;
14801        Runner {
14802            state,
14803            roles: ResolvedRoles {
14804                implementers: Vec::new(),
14805                judges: Vec::new(),
14806                reviewers: Vec::new(),
14807                fixer: None,
14808                conductor: conductor(),
14809                implementer_roster: Vec::new(),
14810                judge_roster: Vec::new(),
14811                reviewer_roster: Vec::new(),
14812            },
14813            sem: Arc::new(Semaphore::new(1)),
14814            pause: Pause::new(),
14815            interrupt: Pause::new(),
14816        }
14817    }
14818
14819    #[tokio::test]
14820    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14821        crate::run::pin_test_home();
14822        let mut runner = runner_at(RunStatus::Gating);
14823        runner.state.save().unwrap();
14824        let id = runner.state.id.clone();
14825        let passes = runner.state.passes.len();
14826        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14827        let mut claiming = Box::pin(runner.claim_driver());
14828        std::future::poll_fn(|cx| {
14829            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14830            std::task::Poll::Ready(())
14831        })
14832        .await;
14833        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14834        drop(owner);
14835        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14836            .await
14837            .unwrap()
14838            .unwrap();
14839        assert_eq!(execution.runner.state.passes.len(), passes + 1);
14840        assert!(execution.runner.state.resource_block.is_none());
14841        assert_eq!(
14842            execution.runner.state.liveness(false),
14843            crate::run::Liveness::Live
14844        );
14845        drop(execution);
14846        assert!(RunState::load(&id).unwrap().driver_exited);
14847    }
14848
14849    #[tokio::test]
14850    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14851        crate::run::pin_test_home();
14852        let mut runner = runner_at(RunStatus::Gating);
14853        runner.state.save().unwrap();
14854        // run.json can commit, but the subsequent lease metadata write fails.
14855        // Cleanup must run after releasing the short ownership lock.
14856        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14857        assert!(runner.claim_driver().await.is_err());
14858        assert!(runner.state.driver_exited);
14859        assert_eq!(runner.state.driver_pid, None);
14860        let disk = RunState::load(&runner.state.id).unwrap();
14861        assert!(disk.driver_exited);
14862        assert_eq!(disk.driver_pid, None);
14863        assert_eq!(disk.status, RunStatus::Gating);
14864    }
14865
14866    #[tokio::test]
14867    async fn an_early_graph_error_releases_its_driver() {
14868        crate::run::pin_test_home();
14869        let mut runner = runner_at(RunStatus::Blocked);
14870        runner.state.config.graph.selected_implementer = Some("selected".into());
14871        runner.state.save().unwrap();
14872        assert!(runner.execute().await.is_err());
14873        let disk = RunState::load(&runner.state.id).unwrap();
14874        assert!(disk.driver_exited);
14875        assert_eq!(disk.driver_pid, None);
14876        assert_eq!(disk.status, RunStatus::Blocked);
14877    }
14878
14879    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14880        crate::run::MergeRepair {
14881            question: "repair-question".into(),
14882            head: "old-head".into(),
14883            pr: "https://github.com/o/r/pull/42".into(),
14884            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14885            gate: vec![outcome(Some(0), false)],
14886            gate_head: Some("old-head".into()),
14887            gate_fixes: Vec::new(),
14888            pre_gate: Vec::new(),
14889            phase,
14890            failure: None,
14891        }
14892    }
14893
14894    #[tokio::test]
14895    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14896        crate::run::pin_test_home();
14897        let mut runner = runner_at(RunStatus::Landing);
14898        runner.state.merge_repairs.push(merge_repair_request(
14899            crate::run::MergeRepairPhase::Requested,
14900        ));
14901        runner.state.save().unwrap();
14902        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14903        assert!(
14904            runner
14905                .execute()
14906                .await
14907                .unwrap_err()
14908                .to_string()
14909                .contains("already running")
14910        );
14911        assert_eq!(
14912            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14913            crate::run::MergeRepairPhase::Requested
14914        );
14915        drop(claim);
14916        let mut disk = runner.state.clone();
14917        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14918        disk.save().unwrap();
14919        assert!(
14920            runner
14921                .execute()
14922                .await
14923                .unwrap_err()
14924                .to_string()
14925                .contains("decision changed")
14926        );
14927        assert_eq!(
14928            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14929            crate::run::MergeRepairPhase::Complete
14930        );
14931    }
14932
14933    #[test]
14934    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14935        crate::run::pin_test_home();
14936        let mut runner = runner_at(RunStatus::Landing);
14937        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14938        runner.state.reviews = request.reviews.clone();
14939        runner.state.gate = request.gate.clone();
14940        runner.state.gate_ran = true;
14941        runner.state.land_approval = Some(crate::run::LandApproval {
14942            question: "old".into(),
14943            head: "old-head".into(),
14944        });
14945        runner.state.merge_repairs.push(request);
14946        runner.begin_merge_repair_review(0).unwrap();
14947        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14948        assert!(!runner.state.gate_status().ok());
14949        assert!(runner.state.land_approval.is_none());
14950        let disk = RunState::load(&runner.state.id).unwrap();
14951        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14952        assert!(disk.merge_repairs[0].gate[0].ok());
14953        assert_eq!(
14954            disk.merge_repairs[0].phase,
14955            crate::run::MergeRepairPhase::Reviewing
14956        );
14957        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14958        assert!(disk.reviews.is_empty());
14959    }
14960
14961    #[tokio::test]
14962    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14963        crate::run::pin_test_home();
14964        for phase in [
14965            crate::run::MergeRepairPhase::Fixing,
14966            crate::run::MergeRepairPhase::Requested,
14967        ] {
14968            let mut runner = runner_at(RunStatus::Landing);
14969            runner.state.config.graph.land_rounds = 0;
14970            runner.state.merge_repairs.push(merge_repair_request(phase));
14971            runner.state.save().unwrap();
14972            assert!(!runner.repair_merge_review().await.unwrap());
14973            assert_eq!(runner.state.status, RunStatus::Blocked);
14974            assert_eq!(
14975                runner.state.merge_repairs[0].phase,
14976                crate::run::MergeRepairPhase::Failed
14977            );
14978            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14979            assert!(
14980                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14981                    "interrupted"
14982                } else {
14983                    "budget exhausted"
14984                })
14985            );
14986            assert!(runner.state.merge.is_none());
14987        }
14988    }
14989
14990    /// `park_here` folding in the reason `Pause::park_because` recorded -
14991    /// this is what lets an operator reading a run's events tell an
14992    /// interrupt-driven park from an ordinary shutdown park.
14993    #[test]
14994    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14995        crate::run::pin_test_home();
14996        let mut runner = runner_at(RunStatus::Implementing);
14997        let interrupt = Pause::new();
14998        runner.watch_interrupt(interrupt.clone());
14999
15000        interrupt.park_because("task a1b2 asked to run first");
15001
15002        assert!(runner.park_here().expect("park_here"));
15003        assert!(runner.state.parked);
15004        let last = runner.state.events.last().expect("a park event");
15005        assert_eq!(last.node, "park");
15006        assert!(
15007            last.message.contains("task a1b2 asked to run first"),
15008            "expected the interrupt reason in {:?}",
15009            last.message
15010        );
15011    }
15012
15013    /// The cause is stored on the pass when it parks, and the first cause a
15014    /// shared pause was given is the one that stays.
15015    #[test]
15016    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15017        crate::run::pin_test_home();
15018        let mut runner = runner_at(RunStatus::Implementing);
15019        runner.state.begin_pass();
15020        let shutdown = Pause::new();
15021        runner.on_pause(shutdown.clone());
15022        let upgrade = ParkCause::Upgrade {
15023            from: "0.121.0".to_owned(),
15024            to: Some("0.121.1".to_owned()),
15025        };
15026        shutdown.park_for(upgrade.clone());
15027        shutdown.park_for(ParkCause::Operator);
15028
15029        assert!(runner.park_here().expect("park_here"));
15030        assert_eq!(
15031            runner.state.passes[0].end,
15032            Some(crate::run::PassEnd::Parked { cause: upgrade })
15033        );
15034        let last = runner.state.events.last().expect("a park event");
15035        assert!(last.message.contains("0.121.0"), "{}", last.message);
15036    }
15037
15038    #[tokio::test]
15039    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15040        crate::run::pin_test_home();
15041        let mut runner = runner_at(RunStatus::Reviewing);
15042        runner.state.save().unwrap();
15043        let mut execution = runner.claim_driver().await.expect("first execution");
15044        assert_eq!(execution.runner.state.passes.len(), 1);
15045        execution
15046            .driver
15047            .finish(&mut execution.runner.state)
15048            .await
15049            .unwrap();
15050    }
15051
15052    #[tokio::test]
15053    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15054        crate::run::pin_test_home();
15055        let mut runner = runner_at(RunStatus::Reviewing);
15056        runner.state.config.agents = vec![conductor()];
15057        runner.state.config.graph.timeout_resource_wait = 0;
15058        runner.state.save().unwrap();
15059        let id = runner.state.id.clone();
15060        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15061        assert!(runner.claim_driver().await.is_err());
15062        drop(owner);
15063        drop(runner);
15064        let mut resumed = Runner::resume(&id).unwrap();
15065        assert_eq!(resumed.state.driver_claimed, Some(false));
15066        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15067        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15068        let mut execution = resumed
15069            .claim_driver()
15070            .await
15071            .expect("retry initial claim after reload");
15072        assert_eq!(execution.runner.state.driver_claimed, Some(true));
15073        assert_eq!(execution.runner.state.passes.len(), 1);
15074        execution
15075            .driver
15076            .finish(&mut execution.runner.state)
15077            .await
15078            .unwrap();
15079    }
15080
15081    #[tokio::test]
15082    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15083        crate::run::pin_test_home();
15084        let mut runner = runner_at(RunStatus::Reviewing);
15085        runner.state.config.agents = vec![conductor()];
15086        runner.state.save().unwrap();
15087        let id = runner.state.id.clone();
15088        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15089        let mut resumed = Runner::resume(&id).unwrap();
15090        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15091        assert!(resumed.claim_driver().await.is_err());
15092        assert!(RunState::load(&id).unwrap().passes.is_empty());
15093        drop(lease);
15094        let mut execution = resumed.claim_driver().await.unwrap();
15095        execution
15096            .driver
15097            .finish(&mut execution.runner.state)
15098            .await
15099            .unwrap();
15100    }
15101
15102    #[tokio::test]
15103    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15104        crate::run::pin_test_home();
15105        let mut runner = runner_at(RunStatus::Reviewing);
15106        let state = &mut runner.state;
15107        state.config.agents = vec![conductor()];
15108        state.driver_claimed = None;
15109        state.driver_pid = Some(std::process::id());
15110        state.driver_started_at = None;
15111        state.begin_pass();
15112        state.save().unwrap();
15113        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15114        std::fs::write(
15115            state.dir().join("driver.json"),
15116            serde_json::to_vec(&serde_json::json!({
15117                "pid": state.driver_pid, "started_at": null,
15118                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15119            }))
15120            .unwrap(),
15121        )
15122        .unwrap();
15123        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15124        drop(lease);
15125        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15126        let mut resumed = Runner::resume(&state.id).unwrap();
15127        assert!(!resumed.state.driver_exited);
15128        let mut execution = resumed
15129            .claim_driver()
15130            .await
15131            .expect("lease proves previous execution ended");
15132        assert_eq!(execution.runner.state.passes.len(), 2);
15133        execution
15134            .driver
15135            .finish(&mut execution.runner.state)
15136            .await
15137            .unwrap();
15138    }
15139
15140    #[tokio::test]
15141    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15142        crate::run::pin_test_home();
15143        let mut runner = runner_at(RunStatus::Reviewing);
15144        let mut json = serde_json::to_value(&runner.state).unwrap();
15145        json.as_object_mut().unwrap().remove("driver_claimed");
15146        runner.state = serde_json::from_value(json).unwrap();
15147        assert_eq!(runner.state.driver_claimed, None);
15148        runner.state.save().unwrap();
15149        let mut execution = runner
15150            .claim_driver()
15151            .await
15152            .expect("the execution lease, not a missing marker, excludes a second driver");
15153        assert_eq!(execution.runner.state.passes.len(), 1);
15154        execution
15155            .driver
15156            .finish(&mut execution.runner.state)
15157            .await
15158            .unwrap();
15159    }
15160
15161    #[tokio::test]
15162    async fn driver_admission_refuses_a_proven_live_driver() {
15163        crate::run::pin_test_home();
15164        let mut runner = runner_at(RunStatus::Reviewing);
15165        let pid = std::process::id();
15166        runner.state.driver_pid = Some(pid);
15167        runner.state.driver_started_at = crate::proc::process_started_at(pid);
15168        assert!(runner.state.driver_started_at.is_some());
15169        runner.state.save().unwrap();
15170        let error = runner
15171            .claim_driver()
15172            .await
15173            .err()
15174            .expect("a live driver must refuse");
15175        assert!(error.to_string().contains("live driver"), "{error:#}");
15176        let saved = RunState::load(&runner.state.id).unwrap();
15177        assert_eq!(saved.driver_pid, Some(pid));
15178        assert_eq!(saved.status, RunStatus::Reviewing);
15179        assert!(saved.passes.is_empty());
15180    }
15181
15182    #[test]
15183    fn parking_resource_halt_persists_the_pass_once() {
15184        crate::run::pin_test_home();
15185        let mut runner = runner_at(RunStatus::Reviewing);
15186        runner.state.begin_pass();
15187        runner.state.resource_block = Some(crate::run::ResourceBlock {
15188            node: "fix".into(),
15189            kind: "shared build cache".into(),
15190            detail: "cancelled".into(),
15191            at: Timestamp::now(),
15192            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15193        });
15194        let cause = ParkCause::Upgrade {
15195            from: "old".into(),
15196            to: Some("new".into()),
15197        };
15198        runner.pause.park_for(cause.clone());
15199        // Both direct fixer exits and graph resource boundaries must complete
15200        // parking, rather than returning a stale Reviewing record.
15201        runner.stop_for_resource().unwrap();
15202        assert!(runner.resource_halt().unwrap());
15203        assert!(runner.park_here().unwrap());
15204        let saved = RunState::load(&runner.state.id).unwrap();
15205        assert!(saved.parked);
15206        assert_eq!(saved.status, RunStatus::Reviewing);
15207        assert!(saved.passes[0].ended_at.is_some());
15208        assert_eq!(
15209            saved.passes[0].end,
15210            Some(crate::run::PassEnd::Parked {
15211                cause: cause.clone()
15212            })
15213        );
15214        assert_eq!(
15215            saved.resource_block.unwrap().cause,
15216            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15217        );
15218        assert_eq!(
15219            saved
15220                .events
15221                .iter()
15222                .filter(|event| event.node == "park")
15223                .count(),
15224            1
15225        );
15226    }
15227
15228    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
15229    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
15230    /// cleared) must not make a *different* run - one only watching its own,
15231    /// unshared interrupt `Pause` - see itself as parked. If a future change
15232    /// ever collapsed these back into one handle, the interrupt scheduler
15233    /// would park every run for the rest of the daemon's life, not just the
15234    /// one it meant to interrupt.
15235    #[test]
15236    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15237        crate::run::pin_test_home();
15238        let mut runner = runner_at(RunStatus::Implementing);
15239        let shutdown = Pause::new();
15240        runner.on_pause(shutdown.clone());
15241        let interrupt = Pause::new();
15242        runner.watch_interrupt(interrupt.clone());
15243
15244        // Nobody has asked for anything yet.
15245        assert!(!runner.park_here().expect("park_here"));
15246        assert!(!runner.state.parked);
15247
15248        // Only the interrupt handle fires; the shutdown handle stays clear.
15249        interrupt.park_because("test");
15250        assert!(!shutdown.parked());
15251        assert!(runner.park_here().expect("park_here"));
15252    }
15253
15254    /// The property every prior attempt at this feature failed to pin down:
15255    /// asking a run to park while one of its nodes has a real, in-flight
15256    /// async operation running (an agent call, in production) must not cut
15257    /// that operation short. `park_here` is only ever consulted *between*
15258    /// `execute`'s node calls - see its own doc - so nothing inside a node
15259    /// can observe a park request until the node itself returns. This proves
15260    /// that structurally, with real `tokio` concurrency and a channel
15261    /// handshake (never a sleep, which would only prove "usually", not
15262    /// "cannot"): the "node" below reports that it has genuinely started,
15263    /// and only then is the park requested; the node still has to be told to
15264    /// finish before `park_here` is ever called, exactly mirroring every
15265    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
15266    /// in `execute`.
15267    #[tokio::test]
15268    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15269        crate::run::pin_test_home();
15270        let mut runner = runner_at(RunStatus::Implementing);
15271        let interrupt = Pause::new();
15272        runner.watch_interrupt(interrupt.clone());
15273
15274        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15275        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15276
15277        // Stands in for one node's in-flight agent call: it proves it has
15278        // genuinely started, then blocks - exactly as a spawned CLI process
15279        // does - until told to finish.
15280        let node = async move {
15281            started_tx.send(()).expect("send started");
15282            finish_rx.await.expect("recv finish");
15283            "node finished"
15284        };
15285
15286        let interrupter = async move {
15287            started_rx.await.expect("recv started");
15288            // The call is now genuinely in flight. Ask it to park.
15289            interrupt.park_because("higher-priority task waiting");
15290            // Nothing the node does can observe this yet - there is no
15291            // check inside it, by construction - so let the executor run
15292            // anything pending and then let the node finish on its own.
15293            tokio::task::yield_now().await;
15294            finish_tx.send(()).expect("send finish");
15295        };
15296
15297        let (node_result, ()) = tokio::join!(node, interrupter);
15298        assert_eq!(
15299            node_result, "node finished",
15300            "the in-flight call ran to completion"
15301        );
15302
15303        // Only now, at the boundary the real `execute` would check right
15304        // after this node, does the park take effect.
15305        assert!(runner.park_here().expect("park_here"));
15306        assert!(runner.state.parked);
15307    }
15308
15309    /// A run parked mid-competition carries every field it had accumulated
15310    /// through the exact same disk round-trip an ordinary resume uses -
15311    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
15312    /// Nothing about parking for an interrupt is a special case of that path;
15313    /// this is what proves it rather than assuming it.
15314    #[test]
15315    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15316        crate::run::pin_test_home();
15317        let mut runner = runner_at(RunStatus::Judging);
15318        // `Runner::resume` re-resolves roles from the saved config, which
15319        // refuses an empty roster - give it the same minimal one `conductor`
15320        // itself uses.
15321        runner.state.config.agents = vec![conductor()];
15322        runner.state.candidates = vec![Candidate {
15323            index: 0,
15324            label: 'A',
15325            agent: "alpha".to_owned(),
15326            branch: "magi/x/A".to_owned(),
15327            worktree: PathBuf::from("/nonexistent/worktree"),
15328            summary: "did the thing".to_owned(),
15329            stat: "1 file changed".to_owned(),
15330            files: 1,
15331            commits: 1,
15332            empty: false,
15333            failed: None,
15334            verified_noop: None,
15335            duration_ms: 1234,
15336            folded: false,
15337        }];
15338        let run_id = runner.state.id.clone();
15339
15340        let interrupt = Pause::new();
15341        runner.watch_interrupt(interrupt.clone());
15342        interrupt.park_because("task c3d4 asked to run first");
15343        assert!(runner.park_here().expect("park_here"));
15344
15345        let resumed = Runner::resume(&run_id).expect("resume");
15346        assert_eq!(resumed.state.candidates.len(), 1);
15347        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15348        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15349        assert_eq!(resumed.state.status, runner.state.status);
15350        assert!(
15351            resumed.state.parked,
15352            "still parked until `execute` actually walks the graph again"
15353        );
15354        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15355    }
15356
15357    /// A fresh open question on `run`, stored and handed back for assertions.
15358    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15359        let mut q = ask::Question::new(
15360            run.to_owned(),
15361            "implement".to_owned(),
15362            "impl-A".to_owned(),
15363            "Which storage backend should the cache use?".to_owned(),
15364            String::new(),
15365            vec!["SQLite".to_owned(), "Redis".to_owned()],
15366        );
15367        store.put(&mut q).unwrap();
15368        q
15369    }
15370
15371    #[test]
15372    fn a_failed_runs_open_question_is_abandoned() {
15373        ask_test_home();
15374        let store = ask::Questions::open();
15375        let mut runner = runner_at(RunStatus::Failed);
15376        let run = runner.state.id.clone();
15377        let q = ask_open_question(&store, &run);
15378
15379        runner.settle_questions();
15380
15381        let back = store.get(&q.id).unwrap();
15382        assert!(
15383            !back.status.open(),
15384            "the seat that asked died with the run; nobody is left to read an answer"
15385        );
15386        assert!(
15387            back.detail.contains(&run) && back.detail.contains("failed"),
15388            "the reason names what the run became, not just that it is gone: {}",
15389            back.detail
15390        );
15391    }
15392
15393    #[test]
15394    fn a_merged_runs_open_question_is_abandoned_too() {
15395        ask_test_home();
15396        let store = ask::Questions::open();
15397        // A run that finishes cleanly still leaves nobody to read an answer -
15398        // this is not only a failure-path cleanup.
15399        for status in [RunStatus::Merged, RunStatus::Ready] {
15400            let mut runner = runner_at(status);
15401            let run = runner.state.id.clone();
15402            let q = ask_open_question(&store, &run);
15403
15404            runner.settle_questions();
15405
15406            let back = store.get(&q.id).unwrap();
15407            assert!(
15408                !back.status.open(),
15409                "{status:?} run's question must not outlive the run"
15410            );
15411        }
15412    }
15413
15414    #[test]
15415    fn a_still_resumable_runs_open_question_is_left_alone() {
15416        ask_test_home();
15417        let store = ask::Questions::open();
15418        // `Blocked` and `Stalled` can still be resumed — the candidates, the
15419        // review round and the seat sessions are all still on disk — so a
15420        // question asked mid-round may yet get a real answer from a real
15421        // resume. Sweeping it here would be exactly the failure mode this
15422        // whole feature exists to avoid on the other side.
15423        for status in [RunStatus::Blocked, RunStatus::Stalled] {
15424            let mut runner = runner_at(status);
15425            let run = runner.state.id.clone();
15426            let q = ask_open_question(&store, &run);
15427
15428            runner.settle_questions();
15429
15430            let back = store.get(&q.id).unwrap();
15431            assert!(
15432                back.status.open(),
15433                "{status:?} is still alive; the question must still be waiting"
15434            );
15435        }
15436    }
15437
15438    #[test]
15439    fn settle_questions_never_touches_an_already_answered_question() {
15440        ask_test_home();
15441        let store = ask::Questions::open();
15442        let mut runner = runner_at(RunStatus::Failed);
15443        let run = runner.state.id.clone();
15444        let mut q = ask_open_question(&store, &run);
15445        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15446            .unwrap();
15447        store.put(&mut q).unwrap();
15448
15449        // Called twice, the way a crash-recovered daemon reclaim and the
15450        // graph's own cleanup both can for the same run — `abandon_for_run`
15451        // only ever touches what is still open, so this must be inert both
15452        // times, not merely the second.
15453        runner.settle_questions();
15454        runner.settle_questions();
15455
15456        let back = store.get(&q.id).unwrap();
15457        assert_eq!(
15458            back.status,
15459            ask::QuestionStatus::Answered,
15460            "a real answer is a decision on record, never overwritten by a sweep"
15461        );
15462    }
15463
15464    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
15465    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
15466    /// without merging, whose winner is still the operator's answer to read.
15467    /// Nothing previously called `fold_run` itself with a real `tally`, so
15468    /// this is the first test to pin down the one distinction the whole
15469    /// automatic-fold feature depends on: the winner's worktree and branch
15470    /// must survive, everything else sharing the run's worktree bay - a
15471    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
15472    /// second sweep treats every non-winner directory under the bay alike -
15473    /// must not.
15474    #[tokio::test]
15475    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15476        crate::run::pin_test_home();
15477        let tmp = tempfile::tempdir().expect("tempdir");
15478        let repo = tmp.path().join("repo");
15479        std::fs::create_dir_all(&repo).unwrap();
15480        init_repo(&repo);
15481
15482        let mut config = Config::default();
15483        config.graph.worktree_root = Some(tmp.path().join("wt"));
15484
15485        let mut state = RunState::new(
15486            repo.clone(),
15487            "main".to_owned(),
15488            "deadbeef".to_owned(),
15489            "task".to_owned(),
15490            config,
15491        );
15492        let root = state.worktree_root();
15493        let wt_a = root.join("cand-A");
15494        let wt_b = root.join("cand-B");
15495        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15496            .await
15497            .expect("worktree A");
15498        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15499            .await
15500            .expect("worktree B");
15501
15502        state.candidates = vec![
15503            Candidate {
15504                index: 0,
15505                label: 'A',
15506                agent: "alpha".to_owned(),
15507                branch: "magi/x/A".to_owned(),
15508                worktree: wt_a.clone(),
15509                summary: String::new(),
15510                stat: String::new(),
15511                files: 0,
15512                commits: 0,
15513                empty: false,
15514                failed: None,
15515                verified_noop: None,
15516                duration_ms: 0,
15517                folded: false,
15518            },
15519            Candidate {
15520                index: 1,
15521                label: 'B',
15522                agent: "beta".to_owned(),
15523                branch: "magi/x/B".to_owned(),
15524                worktree: wt_b.clone(),
15525                summary: String::new(),
15526                stat: String::new(),
15527                files: 0,
15528                commits: 0,
15529                empty: false,
15530                failed: None,
15531                verified_noop: None,
15532                duration_ms: 0,
15533                folded: false,
15534            },
15535        ];
15536        state.tally = Some(Tally {
15537            first_choice: BTreeMap::from([('A', 1)]),
15538            borda: BTreeMap::new(),
15539            winner: 'A',
15540            rankings: 1,
15541            unanimous_initial: true,
15542            deliberated: false,
15543            changed_votes: 0,
15544            unanimous_final: true,
15545            tie_break: None,
15546            judges: 1,
15547            present: 1,
15548            quorum: 1,
15549            met_quorum: true,
15550            uncontested: None,
15551        });
15552        state.status = RunStatus::Ready;
15553
15554        fold_run(&mut state, false, &crate::run::home())
15555            .await
15556            .expect("fold_run");
15557
15558        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15559        assert!(
15560            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15561            "the unmerged winner's branch survives"
15562        );
15563        assert!(
15564            !state.candidates[0].folded,
15565            "the winner is not marked folded"
15566        );
15567
15568        assert!(!wt_b.exists(), "the loser's worktree is removed");
15569        assert!(
15570            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15571            "the loser's branch is removed"
15572        );
15573        assert!(state.candidates[1].folded, "the loser is marked folded");
15574    }
15575
15576    #[tokio::test]
15577    #[cfg(unix)]
15578    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15579        use std::os::fd::AsRawFd as _;
15580        use std::time::Duration;
15581
15582        let tmp = tempfile::tempdir().unwrap();
15583        let repo = tmp.path().join("repo");
15584        std::fs::create_dir_all(&repo).unwrap();
15585        init_repo(&repo);
15586        let common = repo.join(".git");
15587        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15588        let home = tmp.path().join("home");
15589        let mut config = Config::default();
15590        config.coordination = crate::config::test_coordination();
15591        config.graph.worktree_root = Some(tmp.path().join("wt"));
15592        config.graph.timeout_resource_wait = 1;
15593        let mut state = RunState::new(
15594            repo.clone(),
15595            "main".into(),
15596            "deadbeef".into(),
15597            "task".into(),
15598            config,
15599        );
15600        let root = state.worktree_root();
15601        let candidate = root.join("cand-A");
15602        let reviewer = root.join("review-1");
15603        let branch = state.branch_for('A');
15604        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15605            .await
15606            .unwrap();
15607        git::worktree_add_detached(&repo, &reviewer, "main")
15608            .await
15609            .unwrap();
15610        git::acquire_worktree_config(&repo).await.unwrap();
15611        state.enabled_worktree_config = true;
15612        state.status = RunStatus::Ready;
15613        state.candidates.push(Candidate {
15614            index: 0,
15615            label: 'A',
15616            agent: "implement".into(),
15617            branch: branch.clone(),
15618            worktree: candidate.clone(),
15619            summary: String::new(),
15620            stat: String::new(),
15621            files: 0,
15622            commits: 0,
15623            empty: false,
15624            failed: None,
15625            verified_noop: None,
15626            duration_ms: 0,
15627            folded: false,
15628        });
15629        state.save_under(&home).unwrap();
15630        let config_before = std::fs::read(common.join("config")).unwrap();
15631        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15632            .await
15633            .unwrap();
15634        // A separate OS handle bypasses the launcher's local gate, exactly as
15635        // an external updater does, and must still prevent all folding work.
15636        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15637        assert_eq!(
15638            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15639            0
15640        );
15641        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15642        assert!(
15643            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15644                .await
15645                .is_err()
15646        );
15647        assert!(candidate.exists());
15648        assert!(reviewer.exists());
15649        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15650        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15651        assert_eq!(
15652            git::git(&repo, &["worktree", "list", "--porcelain"])
15653                .await
15654                .unwrap(),
15655            worktrees_before
15656        );
15657        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15658            .await
15659            .unwrap()
15660            .unwrap_err();
15661        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15662        drop(fold);
15663        assert!(candidate.exists() && reviewer.exists());
15664        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15665        assert_eq!(
15666            state.resource_block.as_ref().unwrap().kind,
15667            "repository coordination"
15668        );
15669        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15670        drop(external);
15671        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15672        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15673            .await
15674            .unwrap()
15675            .unwrap();
15676        drop(fold);
15677        assert!(removed.contains(&branch));
15678        assert!(!root.exists());
15679        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15680        assert!(state.candidates[0].folded);
15681        assert!(!state.enabled_worktree_config);
15682        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15683        let enabled = git::git_raw(
15684            &repo,
15685            &["config", "--local", "--get", "extensions.worktreeConfig"],
15686        )
15687        .await
15688        .unwrap();
15689        assert!(!enabled.ok(), "the last config reference was released");
15690    }
15691
15692    /// A branch handed to a later run is that run's (and its pull request's):
15693    /// folding the run that released it must not delete it.
15694    #[tokio::test]
15695    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15696        let tmp = tempfile::tempdir().expect("tempdir");
15697        let repo = tmp.path().join("repo");
15698        std::fs::create_dir_all(&repo).unwrap();
15699        init_repo(&repo);
15700        let home = tmp.path().join("home");
15701
15702        let mut config = Config::default();
15703        config.graph.worktree_root = Some(tmp.path().join("wt"));
15704        let mut state = RunState::new(
15705            repo.clone(),
15706            "main".to_owned(),
15707            "deadbeef".to_owned(),
15708            "task".to_owned(),
15709            config,
15710        );
15711        // The worktree is already gone (released); the branch survives.
15712        git::git(&repo, &["branch", "magi/x/A", "main"])
15713            .await
15714            .expect("branch");
15715        state.candidates = vec![Candidate {
15716            index: 0,
15717            label: 'A',
15718            agent: "alpha".to_owned(),
15719            branch: "magi/x/A".to_owned(),
15720            worktree: state.worktree_root().join("cand-A"),
15721            summary: String::new(),
15722            stat: String::new(),
15723            files: 0,
15724            commits: 0,
15725            empty: false,
15726            failed: None,
15727            verified_noop: None,
15728            duration_ms: 0,
15729            folded: true,
15730        }];
15731        state.released_to = Some("20260901-000000-new1".to_owned());
15732        state.released_branches = vec!["magi/x/A".to_owned()];
15733
15734        fold_run(&mut state, true, &home).await.expect("fold_run");
15735        state.status = RunStatus::Merged;
15736        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15737        state.merge = Some(crate::run::MergeOutcome {
15738            mode: MergeMode::Pr,
15739            ok: true,
15740            detail: "confirmed merged".into(),
15741            empty: false,
15742        });
15743        fold_run(&mut state, true, &home)
15744            .await
15745            .expect("merged fold");
15746
15747        assert!(
15748            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15749            "the handed-over branch survives a fold"
15750        );
15751    }
15752
15753    /// A winner with nothing ahead of the base is caught before `gh` is ever
15754    /// asked for a pull request, and the message carries what the task's
15755    /// references resolved to.
15756    #[tokio::test]
15757    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15758        let tmp = tempfile::tempdir().expect("tempdir");
15759        let repo = tmp.path().join("repo");
15760        std::fs::create_dir_all(&repo).unwrap();
15761        init_repo(&repo);
15762        let run = |args: &[&str]| {
15763            let out = std::process::Command::new("git")
15764                .quiet()
15765                .args(args)
15766                .current_dir(&repo)
15767                .output()
15768                .expect("spawn git");
15769            assert!(out.status.success(), "git {args:?}");
15770        };
15771        run(&["branch", "magi/x/A"]);
15772        run(&["checkout", "-q", "-b", "magi/x/B"]);
15773        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15774        run(&["add", "-A"]);
15775        run(&["commit", "-q", "-m", "work"]);
15776        run(&["checkout", "-q", "main"]);
15777
15778        // A pull request is compared against the remote's base, so the
15779        // fixture needs one. Before it exists nothing can be read, and the
15780        // answer must be "not empty".
15781        let probe = RunState::new(
15782            repo.clone(),
15783            "main".to_owned(),
15784            "deadbeef".to_owned(),
15785            "task".to_owned(),
15786            Config::default(),
15787        );
15788        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15789        let bare = tmp.path().join("origin.git");
15790        let out = std::process::Command::new("git")
15791            .quiet()
15792            .args(["init", "-q", "--bare"])
15793            .arg(&bare)
15794            .output()
15795            .expect("spawn git");
15796        assert!(out.status.success(), "git init --bare");
15797        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15798        run(&["push", "-q", "origin", "main"]);
15799
15800        let mut state = RunState::new(
15801            repo.clone(),
15802            "main".to_owned(),
15803            "deadbeef".to_owned(),
15804            "task".to_owned(),
15805            Config::default(),
15806        );
15807        state.seeds = vec![refs::Seed {
15808            token: "magi/27b2/A".to_owned(),
15809            kind: refs::SeedKind::Unresolved,
15810            sha: String::new(),
15811            branch: true,
15812            detail: "no branch or commit named magi/27b2/A".to_owned(),
15813        }];
15814
15815        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15816        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15817        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15818        let detail = empty_candidate_detail(&state, "main");
15819        assert!(detail.starts_with("empty candidate"), "{detail}");
15820        assert!(!detail.contains("magi/27b2/A"), "{detail}");
15821        assert!(detail.contains("--start-from"), "{detail}");
15822    }
15823
15824    /// `status == Ready` used to be read as "this is the harmless
15825    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15826    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15827    /// run's PR was closed without merging — and reentering `merge` with
15828    /// `mode` still `Pr` does not know the difference, so it pushed and
15829    /// opened a second pull request. `mode == Local` reproduces the same
15830    /// blind spot without a network call: reentry must not attempt another
15831    /// git merge once this node has already recorded an outcome.
15832    #[tokio::test]
15833    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15834        let tmp = tempfile::tempdir().expect("tempdir");
15835        let repo = tmp.path().join("repo");
15836        std::fs::create_dir_all(&repo).unwrap();
15837        init_repo(&repo);
15838
15839        let mut config = Config::default();
15840        config.merge.mode = MergeMode::Local;
15841
15842        let mut state = RunState::new(
15843            repo.clone(),
15844            "main".to_owned(),
15845            "deadbeef".to_owned(),
15846            "task".to_owned(),
15847            config,
15848        );
15849        state.candidates = vec![Candidate {
15850            index: 0,
15851            label: 'A',
15852            agent: "alpha".to_owned(),
15853            branch: "does-not-exist".to_owned(),
15854            worktree: repo.clone(),
15855            summary: String::new(),
15856            stat: String::new(),
15857            files: 0,
15858            commits: 0,
15859            empty: false,
15860            failed: None,
15861            verified_noop: None,
15862            duration_ms: 0,
15863            folded: false,
15864        }];
15865        state.tally = Some(Tally {
15866            first_choice: BTreeMap::from([('A', 1)]),
15867            borda: BTreeMap::new(),
15868            winner: 'A',
15869            rankings: 1,
15870            unanimous_initial: true,
15871            deliberated: false,
15872            changed_votes: 0,
15873            unanimous_final: true,
15874            tie_break: None,
15875            judges: 0,
15876            present: 0,
15877            quorum: 0,
15878            met_quorum: true,
15879            uncontested: Some("only candidate A produced a change".to_owned()),
15880        });
15881        state.reviews = vec![ReviewRound {
15882            round: 1,
15883            head: "deadbeef".to_owned(),
15884            verified_head: None,
15885            verified_at: None,
15886            reviews: Vec::new(),
15887            e2e: Vec::new(),
15888            fix: None,
15889            blocking: 0,
15890            answered: 0,
15891            expected: 0,
15892            clean: true,
15893            verify_retried: false,
15894            e2e_deferred: false,
15895            e2e_defer_reason: None,
15896            progressed: false,
15897            vote_split: false,
15898            reconsideration: Vec::new(),
15899            verdict: None,
15900        }];
15901        state.gate = vec![CommandOutcome {
15902            command: "test".to_owned(),
15903            code: Some(0),
15904            output_tail: String::new(),
15905            duration_ms: 0,
15906            resource_blocked: false,
15907        }];
15908        state.gate_ran = true;
15909        // Reached its conclusion already — e.g. `land` closing the PR without
15910        // merging it, which (like the honest `MergeMode::None` path) leaves
15911        // `status` at `Ready`. The recorded outcome is what actually marks
15912        // this node done.
15913        state.status = RunStatus::Ready;
15914        state.merge = Some(MergeOutcome {
15915            mode: MergeMode::Local,
15916            ok: false,
15917            detail: "already concluded".to_owned(),
15918            empty: false,
15919        });
15920
15921        let mut runner = Runner {
15922            state,
15923            roles: ResolvedRoles {
15924                implementers: Vec::new(),
15925                judges: Vec::new(),
15926                reviewers: Vec::new(),
15927                fixer: None,
15928                conductor: conductor(),
15929                implementer_roster: Vec::new(),
15930                judge_roster: Vec::new(),
15931                reviewer_roster: Vec::new(),
15932            },
15933            sem: Arc::new(Semaphore::new(1)),
15934            pause: Pause::new(),
15935            interrupt: Pause::new(),
15936        };
15937
15938        runner.merge().await.expect("merge");
15939
15940        assert_eq!(
15941            runner.state.status,
15942            RunStatus::Ready,
15943            "a concluded run's status must not change on reentry"
15944        );
15945        assert_eq!(
15946            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15947            Some("already concluded"),
15948            "merge must not run again once the node already recorded an outcome"
15949        );
15950    }
15951
15952    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15953    /// when its last attempt was resource-blocked (the shared build cache
15954    /// could not be acquired or confirmed fresh in time - see
15955    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15956    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15957    /// a run merge a tree the gate never actually checked - exactly the case
15958    /// a contended cache produces on every retry until it clears. `merge`
15959    /// must refuse until `gate` has actually recorded an attempt.
15960    #[tokio::test]
15961    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15962        let tmp = tempfile::tempdir().expect("tempdir");
15963        let repo = tmp.path().join("repo");
15964        std::fs::create_dir_all(&repo).unwrap();
15965        init_repo(&repo);
15966
15967        let mut config = Config::default();
15968        config.merge.mode = MergeMode::Local;
15969
15970        let mut state = RunState::new(
15971            repo.clone(),
15972            "main".to_owned(),
15973            "deadbeef".to_owned(),
15974            "task".to_owned(),
15975            config,
15976        );
15977        state.candidates = vec![Candidate {
15978            index: 0,
15979            label: 'A',
15980            agent: "alpha".to_owned(),
15981            branch: "does-not-exist".to_owned(),
15982            worktree: repo.clone(),
15983            summary: String::new(),
15984            stat: String::new(),
15985            files: 0,
15986            commits: 0,
15987            empty: false,
15988            failed: None,
15989            verified_noop: None,
15990            duration_ms: 0,
15991            folded: false,
15992        }];
15993        state.tally = Some(Tally {
15994            first_choice: BTreeMap::from([('A', 1)]),
15995            borda: BTreeMap::new(),
15996            winner: 'A',
15997            rankings: 1,
15998            unanimous_initial: true,
15999            deliberated: false,
16000            changed_votes: 0,
16001            unanimous_final: true,
16002            tie_break: None,
16003            judges: 0,
16004            present: 0,
16005            quorum: 0,
16006            met_quorum: true,
16007            uncontested: Some("only candidate A produced a change".to_owned()),
16008        });
16009        state.reviews = vec![ReviewRound {
16010            round: 1,
16011            head: "deadbeef".to_owned(),
16012            verified_head: None,
16013            verified_at: None,
16014            reviews: Vec::new(),
16015            e2e: Vec::new(),
16016            fix: None,
16017            blocking: 0,
16018            answered: 0,
16019            expected: 0,
16020            clean: true,
16021            verify_retried: false,
16022            e2e_deferred: false,
16023            e2e_defer_reason: None,
16024            progressed: false,
16025            vote_split: false,
16026            reconsideration: Vec::new(),
16027            verdict: None,
16028        }];
16029        // The point: `gate` has not recorded anything yet.
16030        state.gate = Vec::new();
16031        state.gate_ran = false;
16032        state.status = RunStatus::Gating;
16033
16034        let mut runner = Runner {
16035            state,
16036            roles: ResolvedRoles {
16037                implementers: Vec::new(),
16038                judges: Vec::new(),
16039                reviewers: Vec::new(),
16040                fixer: None,
16041                conductor: conductor(),
16042                implementer_roster: Vec::new(),
16043                judge_roster: Vec::new(),
16044                reviewer_roster: Vec::new(),
16045            },
16046            sem: Arc::new(Semaphore::new(1)),
16047            pause: Pause::new(),
16048            interrupt: Pause::new(),
16049        };
16050
16051        runner.merge().await.expect("merge");
16052
16053        assert!(
16054            runner.state.merge.is_none(),
16055            "an empty gate must never be read as a passing one: {:?}",
16056            runner.state.merge
16057        );
16058    }
16059
16060    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
16061    /// commands configured and `merge.mode` is `none` (a review-only run).
16062    /// `gate` must still record a real attempt — zero commands, vacuously
16063    /// passed — rather than leaving `state.gate` empty in a way `merge`
16064    /// cannot tell apart from "never ran"; otherwise the run reaches
16065    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
16066    #[tokio::test]
16067    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16068        let tmp = tempfile::tempdir().expect("tempdir");
16069        let repo = tmp.path().join("repo");
16070        std::fs::create_dir_all(&repo).unwrap();
16071        init_repo(&repo);
16072
16073        // Default config: `verify.gate` empty, `merge.mode` is `none`.
16074        let config = Config::default();
16075
16076        let mut state = RunState::new(
16077            repo.clone(),
16078            "main".to_owned(),
16079            "deadbeef".to_owned(),
16080            "task".to_owned(),
16081            config,
16082        );
16083        state.candidates = vec![Candidate {
16084            index: 0,
16085            label: 'A',
16086            agent: "alpha".to_owned(),
16087            branch: "does-not-exist".to_owned(),
16088            worktree: repo.clone(),
16089            summary: String::new(),
16090            stat: String::new(),
16091            files: 0,
16092            commits: 0,
16093            empty: false,
16094            failed: None,
16095            verified_noop: None,
16096            duration_ms: 0,
16097            folded: false,
16098        }];
16099        state.tally = Some(Tally {
16100            first_choice: BTreeMap::from([('A', 1)]),
16101            borda: BTreeMap::new(),
16102            winner: 'A',
16103            rankings: 1,
16104            unanimous_initial: true,
16105            deliberated: false,
16106            changed_votes: 0,
16107            unanimous_final: true,
16108            tie_break: None,
16109            judges: 0,
16110            present: 0,
16111            quorum: 0,
16112            met_quorum: true,
16113            uncontested: Some("only candidate A produced a change".to_owned()),
16114        });
16115        state.reviews = vec![ReviewRound {
16116            round: 1,
16117            head: "deadbeef".to_owned(),
16118            verified_head: None,
16119            verified_at: None,
16120            reviews: Vec::new(),
16121            e2e: Vec::new(),
16122            fix: None,
16123            blocking: 0,
16124            answered: 0,
16125            expected: 0,
16126            clean: true,
16127            verify_retried: false,
16128            e2e_deferred: false,
16129            e2e_defer_reason: None,
16130            progressed: false,
16131            vote_split: false,
16132            reconsideration: Vec::new(),
16133            verdict: None,
16134        }];
16135
16136        let mut runner = Runner {
16137            state,
16138            roles: ResolvedRoles {
16139                implementers: Vec::new(),
16140                judges: Vec::new(),
16141                reviewers: Vec::new(),
16142                fixer: None,
16143                conductor: conductor(),
16144                implementer_roster: Vec::new(),
16145                judge_roster: Vec::new(),
16146                reviewer_roster: Vec::new(),
16147            },
16148            sem: Arc::new(Semaphore::new(1)),
16149            pause: Pause::new(),
16150            interrupt: Pause::new(),
16151        };
16152
16153        runner.gate().await.expect("gate");
16154        assert!(
16155            runner.state.gate_ran,
16156            "zero configured commands is still a real attempt, not an unrun gate"
16157        );
16158        assert!(runner.state.gate.is_empty());
16159        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16160        assert_ne!(
16161            runner.state.status,
16162            RunStatus::Blocked,
16163            "a gate with nothing to check must not read as failed"
16164        );
16165
16166        runner.merge().await.expect("merge");
16167        assert_eq!(
16168            runner.state.status,
16169            RunStatus::Ready,
16170            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16171        );
16172    }
16173
16174    /// `Config::cache_dir` is derived from `verify.e2e` as well as
16175    /// `verify.gate` (so the e2e leg and the final gate never build against
16176    /// different directories). With zero `verify.gate` commands but a
16177    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
16178    /// that lease before discovering it had nothing to run - so a repo with
16179    /// no gate commands could come back `resource_blocked` (and therefore
16180    /// still `gate_ran == false`) on nothing but an unrelated run holding the
16181    /// cache, exactly the contention this run's own zero commands could
16182    /// never have touched. `gate` must recognise there is nothing to check
16183    /// before it ever asks for the lease.
16184    #[tokio::test]
16185    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16186        crate::run::pin_test_home();
16187        let home = crate::run::home();
16188
16189        let tmp = tempfile::tempdir().expect("tempdir");
16190        let repo = tmp.path().join("repo");
16191        std::fs::create_dir_all(&repo).unwrap();
16192        init_repo(&repo);
16193        // Unique to this test, so holding its lease cannot collide with
16194        // another test sharing the same process-wide `home`.
16195        let cache_dir = tmp.path().join("target");
16196
16197        let mut config = Config::default();
16198        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16199        // `verify.gate` stays empty (the default). Bounded so a regression
16200        // that does start waiting fails the test in seconds, not hangs it.
16201        config.graph.timeout_verify = Some(2);
16202        config.graph.timeout_resource_wait = 2;
16203
16204        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16205        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16206            .expect("no io error acquiring directly")
16207        {
16208            crate::cache::AcquireOutcome::Acquired(g) => g,
16209            crate::cache::AcquireOutcome::Busy(b) => {
16210                panic!("expected the direct acquire to win the lease first: {b:?}")
16211            }
16212        };
16213
16214        let mut state = RunState::new(
16215            repo.clone(),
16216            "main".to_owned(),
16217            "deadbeef".to_owned(),
16218            "task".to_owned(),
16219            config,
16220        );
16221        state.candidates = vec![Candidate {
16222            index: 0,
16223            label: 'A',
16224            agent: "alpha".to_owned(),
16225            branch: "does-not-exist".to_owned(),
16226            worktree: repo.clone(),
16227            summary: String::new(),
16228            stat: String::new(),
16229            files: 0,
16230            commits: 0,
16231            empty: false,
16232            failed: None,
16233            verified_noop: None,
16234            duration_ms: 0,
16235            folded: false,
16236        }];
16237        state.tally = Some(Tally {
16238            first_choice: BTreeMap::from([('A', 1)]),
16239            borda: BTreeMap::new(),
16240            winner: 'A',
16241            rankings: 1,
16242            unanimous_initial: true,
16243            deliberated: false,
16244            changed_votes: 0,
16245            unanimous_final: true,
16246            tie_break: None,
16247            judges: 0,
16248            present: 0,
16249            quorum: 0,
16250            met_quorum: true,
16251            uncontested: Some("only candidate A produced a change".to_owned()),
16252        });
16253        state.reviews = vec![ReviewRound {
16254            round: 1,
16255            head: "deadbeef".to_owned(),
16256            verified_head: None,
16257            verified_at: None,
16258            reviews: Vec::new(),
16259            e2e: Vec::new(),
16260            fix: None,
16261            blocking: 0,
16262            answered: 0,
16263            expected: 0,
16264            clean: true,
16265            verify_retried: false,
16266            e2e_deferred: false,
16267            e2e_defer_reason: None,
16268            progressed: false,
16269            vote_split: false,
16270            reconsideration: Vec::new(),
16271            verdict: None,
16272        }];
16273
16274        let mut runner = Runner {
16275            state,
16276            roles: ResolvedRoles {
16277                implementers: Vec::new(),
16278                judges: Vec::new(),
16279                reviewers: Vec::new(),
16280                fixer: None,
16281                conductor: conductor(),
16282                implementer_roster: Vec::new(),
16283                judge_roster: Vec::new(),
16284                reviewer_roster: Vec::new(),
16285            },
16286            sem: Arc::new(Semaphore::new(1)),
16287            pause: Pause::new(),
16288            interrupt: Pause::new(),
16289        };
16290
16291        let started = std::time::Instant::now();
16292        runner.gate().await.expect("gate");
16293        assert!(
16294            started.elapsed() < Duration::from_secs(1),
16295            "a gate with nothing to run must never wait on a lease it never needed"
16296        );
16297        assert!(
16298            runner.state.gate_ran,
16299            "zero commands is still a real, immediate attempt"
16300        );
16301        assert!(runner.state.gate.is_empty());
16302        assert_ne!(
16303            runner.state.status,
16304            RunStatus::Blocked,
16305            "must not read as resource-blocked on a lease it never asked for"
16306        );
16307    }
16308
16309    #[cfg(unix)]
16310    #[tokio::test]
16311    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16312        crate::run::pin_test_home();
16313        let tmp = tempfile::tempdir().unwrap();
16314        let mut config = Config::default();
16315        config.disk.min_free_bytes = 0;
16316        let mut state = RunState::new(
16317            tmp.path().to_owned(),
16318            "main".into(),
16319            "base".into(),
16320            "verification evidence".into(),
16321            config,
16322        );
16323        let (outcomes, pids) = run_commands(
16324            &mut state,
16325            "gate",
16326            "gate",
16327            1,
16328            &["sh".into(), "-c".into()],
16329            &[
16330                "printf before-timeout; exec sleep 30".into(),
16331                "kill -TERM $$".into(),
16332                "exit 7".into(),
16333                "printf success".into(),
16334            ],
16335            tmp.path(),
16336            Duration::from_secs(1),
16337        )
16338        .await;
16339        wait_for_timed_out_children_to_die(&pids).await;
16340        assert_eq!(
16341            pids.len(),
16342            1,
16343            "unleased timeout retains its death-check PID"
16344        );
16345        assert_eq!(outcomes.len(), 4);
16346        assert_eq!(outcomes[0].code, None);
16347        assert!(outcomes[0].output_tail.contains("before-timeout"));
16348        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16349        assert_eq!(outcomes[1].code, None);
16350        assert!(outcomes[1].output_tail.contains("command terminated:"));
16351        assert!(!outcomes[1].output_tail.contains("timed out"));
16352        assert_eq!(outcomes[2].code, Some(7));
16353        assert!(!outcomes[2].output_tail.contains("timed out"));
16354        assert_eq!(outcomes[3].code, Some(0));
16355        assert_eq!(outcomes[3].output_tail, "success");
16356        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16357    }
16358
16359    /// The addendum's second gap: a `verify.gate` command running for real
16360    /// wall-clock time had nothing at all to show for it in `active` before
16361    /// `run_commands` learned to record it — a run could sit in `Gating` for
16362    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
16363    /// what was actually happening. Proven with a genuinely still-running
16364    /// command, not just a before/after check on the final state: a poller
16365    /// task reads the same `run.json` `gate()` is writing, the same way the
16366    /// phone or `magi show` would, while the shell command is still blocked
16367    /// on its own release marker.
16368    #[tokio::test]
16369    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16370        crate::run::pin_test_home();
16371
16372        let tmp = tempfile::tempdir().expect("tempdir");
16373        let repo = tmp.path().join("repo");
16374        std::fs::create_dir_all(&repo).unwrap();
16375        init_repo(&repo);
16376
16377        let mut config = Config::default();
16378        config.verify.gate = vec![
16379            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16380             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16381                .to_owned(),
16382        ];
16383
16384        let mut state = RunState::new(
16385            repo.clone(),
16386            "main".to_owned(),
16387            "deadbeef".to_owned(),
16388            "task".to_owned(),
16389            config,
16390        );
16391        let run_id = state.id.clone();
16392        state.candidates = vec![Candidate {
16393            index: 0,
16394            label: 'A',
16395            agent: "alpha".to_owned(),
16396            branch: "does-not-exist".to_owned(),
16397            worktree: repo.clone(),
16398            summary: String::new(),
16399            stat: String::new(),
16400            files: 0,
16401            commits: 0,
16402            empty: false,
16403            failed: None,
16404            verified_noop: None,
16405            duration_ms: 0,
16406            folded: false,
16407        }];
16408        state.tally = Some(Tally {
16409            first_choice: BTreeMap::from([('A', 1)]),
16410            borda: BTreeMap::new(),
16411            winner: 'A',
16412            rankings: 1,
16413            unanimous_initial: true,
16414            deliberated: false,
16415            changed_votes: 0,
16416            unanimous_final: true,
16417            tie_break: None,
16418            judges: 0,
16419            present: 0,
16420            quorum: 0,
16421            met_quorum: true,
16422            uncontested: Some("only candidate A produced a change".to_owned()),
16423        });
16424        state.reviews = vec![ReviewRound {
16425            round: 1,
16426            head: "deadbeef".to_owned(),
16427            verified_head: None,
16428            verified_at: None,
16429            reviews: Vec::new(),
16430            e2e: Vec::new(),
16431            fix: None,
16432            blocking: 0,
16433            answered: 0,
16434            expected: 0,
16435            clean: true,
16436            verify_retried: false,
16437            e2e_deferred: false,
16438            e2e_defer_reason: None,
16439            progressed: false,
16440            vote_split: false,
16441            reconsideration: Vec::new(),
16442            verdict: None,
16443        }];
16444
16445        let mut runner = Runner {
16446            state,
16447            roles: ResolvedRoles {
16448                implementers: Vec::new(),
16449                judges: Vec::new(),
16450                reviewers: Vec::new(),
16451                fixer: None,
16452                conductor: conductor(),
16453                implementer_roster: Vec::new(),
16454                judge_roster: Vec::new(),
16455                reviewer_roster: Vec::new(),
16456            },
16457            sem: Arc::new(Semaphore::new(1)),
16458            pause: Pause::new(),
16459            interrupt: Pause::new(),
16460        };
16461
16462        let started_marker = repo.join("started.marker");
16463        let release_marker = repo.join("release.marker");
16464        let poller = tokio::spawn(async move {
16465            // Bounded so a regression that never records the task entry
16466            // fails this test in seconds instead of hanging the suite —
16467            // the same shape `a_park_requested_while_a_seat_is_mid_call_
16468            // does_not_cut_it_short` uses for the same reason.
16469            for _ in 0..100 {
16470                if started_marker.exists()
16471                    && let Ok(s) = crate::run::RunState::load(&run_id)
16472                    && let Some(a) = s.active.get("gate")
16473                {
16474                    std::fs::write(&release_marker, b"go").expect("release marker");
16475                    return Some(a.clone());
16476                }
16477                tokio::time::sleep(Duration::from_millis(50)).await;
16478            }
16479            None
16480        });
16481
16482        runner.gate().await.expect("gate");
16483        let captured = poller.await.expect("poller task");
16484        let captured = captured.expect(
16485            "the poller never saw a `gate` task entry in run.json while the command was \
16486             still blocked on its own release marker",
16487        );
16488
16489        assert_eq!(captured.task.as_deref(), Some("gate"));
16490        assert_eq!(captured.node, "gate");
16491        assert_eq!(captured.index, Some(1));
16492        assert_eq!(captured.total, Some(1));
16493        assert!(
16494            captured
16495                .command
16496                .as_deref()
16497                .is_some_and(|c| c.contains("started.marker")),
16498            "{captured:?}"
16499        );
16500
16501        assert!(
16502            runner.state.active.is_empty(),
16503            "the entry must be cleared once the command actually finished: {:?}",
16504            runner.state.active
16505        );
16506        assert!(runner.state.gate_ran);
16507        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16508    }
16509
16510    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16511    /// record for `land`; one with only a Minor, or no reject, leaves none.
16512    #[tokio::test]
16513    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16514        use crate::verdict::{Finding, ReviewVote, Severity};
16515        crate::run::pin_test_home();
16516        let tmp = tempfile::tempdir().expect("tempdir");
16517        let repo = tmp.path().join("repo");
16518        std::fs::create_dir_all(&repo).unwrap();
16519        init_repo(&repo);
16520
16521        for (severity, vote, expect) in [
16522            (Severity::Major, ReviewVote::Reject, true),
16523            (Severity::Minor, ReviewVote::Reject, false),
16524            (Severity::Major, ReviewVote::Approve, false),
16525        ] {
16526            let mut round = review_round(false, 1, 1, 1, false, true);
16527            round.reviews = vec![ReviewRecord {
16528                reviewer: 1,
16529                agent: "alpha".to_owned(),
16530                summary: String::new(),
16531                findings: vec![Finding {
16532                    id: "R1-1-1".to_owned(),
16533                    severity,
16534                    file: None,
16535                    line: None,
16536                    title: "t".to_owned(),
16537                    detail: String::new(),
16538                }],
16539                vote: Some(vote),
16540                failed: None,
16541                duration_ms: 0,
16542                attempts: 0,
16543            }];
16544            let mut state = RunState::new(
16545                repo.clone(),
16546                "main".to_owned(),
16547                "deadbeef".to_owned(),
16548                "task".to_owned(),
16549                Config::default(),
16550            );
16551            state.reviews = vec![round];
16552            let mut runner = Runner {
16553                state,
16554                roles: ResolvedRoles {
16555                    implementers: Vec::new(),
16556                    judges: Vec::new(),
16557                    reviewers: Vec::new(),
16558                    fixer: None,
16559                    conductor: conductor(),
16560                    implementer_roster: Vec::new(),
16561                    judge_roster: Vec::new(),
16562                    reviewer_roster: Vec::new(),
16563                },
16564                sem: Arc::new(Semaphore::new(1)),
16565                pause: Pause::new(),
16566                interrupt: Pause::new(),
16567            };
16568            let shell = runner.state.config.shell();
16569            runner
16570                .stop_reviewing("round budget spent", &shell, &repo)
16571                .await
16572                .expect("stop_reviewing");
16573            assert_eq!(runner.state.status, RunStatus::Gating);
16574            assert_eq!(
16575                runner.state.contested_handoff.is_some(),
16576                expect,
16577                "{severity:?} + {vote:?}"
16578            );
16579        }
16580    }
16581
16582    /// The shape the incident this whole fix responds to actually had: the
16583    /// round budget spent, the last round's own e2e blocked on the shared
16584    /// build cache (held here by a live pid — this test process — exactly
16585    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16586    /// without forking a process). `stop_reviewing` must retry it — not
16587    /// silently leave the round looking untouched (the catch-up-only half of
16588    /// the bug), and not read the contention as a red `e2e` and block the
16589    /// run on it (the other half). Called directly, the same way
16590    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16591    /// above exercises `gate`, so this never needs a real cargo build to
16592    /// reach: the lease is never released, so `with_cache_lease` never gets
16593    /// past acquiring it into anything that would need a real workspace.
16594    #[tokio::test]
16595    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16596        crate::run::pin_test_home();
16597        let home = crate::run::home();
16598
16599        let tmp = tempfile::tempdir().expect("tempdir");
16600        let repo = tmp.path().join("repo");
16601        std::fs::create_dir_all(&repo).unwrap();
16602        init_repo(&repo);
16603        let head = crate::git::rev_parse(&repo, "HEAD")
16604            .await
16605            .expect("rev-parse");
16606        // Unique to this test, so holding its lease cannot collide with
16607        // another test sharing the same process-wide `home`.
16608        let cache_dir = tmp.path().join("target");
16609
16610        let mut config = Config::default();
16611        config.verify.e2e = vec![format!(
16612            "CARGO_TARGET_DIR='{}' test -f README.md",
16613            cache_dir.display()
16614        )];
16615        config.graph.review_rounds = 1;
16616        // Bounded so a regression that does start waiting fails the test in
16617        // seconds, not hangs it.
16618        config.graph.timeout_verify = Some(2);
16619        config.graph.timeout_resource_wait = 2;
16620
16621        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16622        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16623            .expect("no io error acquiring directly")
16624        {
16625            crate::cache::AcquireOutcome::Acquired(g) => g,
16626            crate::cache::AcquireOutcome::Busy(b) => {
16627                panic!("expected the direct acquire to win the lease first: {b:?}")
16628            }
16629        };
16630
16631        let mut state = RunState::new(
16632            repo.clone(),
16633            "main".to_owned(),
16634            head.clone(),
16635            "task".to_owned(),
16636            config,
16637        );
16638        state.candidates = vec![Candidate {
16639            index: 0,
16640            label: 'A',
16641            agent: "alpha".to_owned(),
16642            branch: "does-not-exist".to_owned(),
16643            worktree: repo.clone(),
16644            summary: String::new(),
16645            stat: String::new(),
16646            files: 0,
16647            commits: 0,
16648            empty: false,
16649            failed: None,
16650            verified_noop: None,
16651            duration_ms: 0,
16652            folded: false,
16653        }];
16654        state.tally = Some(Tally {
16655            first_choice: BTreeMap::from([('A', 1)]),
16656            borda: BTreeMap::new(),
16657            winner: 'A',
16658            rankings: 1,
16659            unanimous_initial: true,
16660            deliberated: false,
16661            changed_votes: 0,
16662            unanimous_final: true,
16663            tie_break: None,
16664            judges: 0,
16665            present: 0,
16666            quorum: 0,
16667            met_quorum: true,
16668            uncontested: Some("only candidate A produced a change".to_owned()),
16669        });
16670        // The round budget's last round, deferred: `needs_catchup_run`'s
16671        // other trigger. `stop_reviewing`'s retry machinery must treat this
16672        // exactly like a resource-blocked attempt once it actually runs.
16673        state.reviews = vec![ReviewRound {
16674            round: 1,
16675            head: head.clone(),
16676            verified_head: None,
16677            verified_at: None,
16678            reviews: Vec::new(),
16679            e2e: Vec::new(),
16680            fix: None,
16681            blocking: 1,
16682            answered: 1,
16683            expected: 1,
16684            clean: false,
16685            verify_retried: false,
16686            e2e_deferred: true,
16687            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16688            progressed: false,
16689            vote_split: false,
16690            reconsideration: Vec::new(),
16691            verdict: None,
16692        }];
16693
16694        let mut runner = Runner {
16695            state,
16696            roles: ResolvedRoles {
16697                implementers: Vec::new(),
16698                judges: Vec::new(),
16699                reviewers: Vec::new(),
16700                fixer: None,
16701                conductor: conductor(),
16702                implementer_roster: Vec::new(),
16703                judge_roster: Vec::new(),
16704                reviewer_roster: Vec::new(),
16705            },
16706            sem: Arc::new(Semaphore::new(1)),
16707            pause: Pause::new(),
16708            interrupt: Pause::new(),
16709        };
16710
16711        let shell = runner.state.config.shell();
16712        runner
16713            .stop_reviewing("round budget spent", &shell, &repo)
16714            .await
16715            .expect("stop_reviewing");
16716
16717        let last = runner.state.reviews.last().expect("round record");
16718        assert_eq!(
16719            last.e2e_status(),
16720            E2eStatus::ResourceBlocked,
16721            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16722             failed: {last:?}"
16723        );
16724        assert_eq!(
16725            last.verified_head.as_deref(),
16726            Some(head.as_str()),
16727            "which commit this attempt targeted is known even though nothing finished checking \
16728             it"
16729        );
16730        let first_attempt_at = last
16731            .verified_at
16732            .expect("when this attempt ran is known too");
16733        assert_ne!(
16734            runner.state.status,
16735            RunStatus::Blocked,
16736            "contention is evidence about the machine, not the patch — it must not settle the \
16737             run as blocked: {:?}",
16738            runner.state.status
16739        );
16740        assert!(
16741            !runner
16742                .state
16743                .events
16744                .iter()
16745                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16746            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16747            runner.state.events
16748        );
16749
16750        // The cache is still held: a later reentry must retry the same
16751        // round's verification again — not leave it looking exactly as
16752        // untouched as the first blocked attempt, which is indistinguishable
16753        // from never having tried again at all.
16754        runner
16755            .stop_reviewing("round budget spent", &shell, &repo)
16756            .await
16757            .expect("stop_reviewing retry");
16758        assert_eq!(
16759            runner.state.reviews.len(),
16760            1,
16761            "no new round was started: {:?}",
16762            runner.state.reviews
16763        );
16764        let last = runner.state.reviews.last().expect("round record");
16765        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16766        assert!(
16767            last.verified_at.expect("still known") > first_attempt_at,
16768            "a second reentry must be a fresh attempt, not a stale copy of the first"
16769        );
16770        assert_ne!(runner.state.status, RunStatus::Blocked);
16771
16772        held.release();
16773    }
16774
16775    #[cfg(unix)]
16776    #[tokio::test]
16777    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
16778        ask_test_home();
16779        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
16780            let tmp = tempfile::tempdir().unwrap();
16781            let repo = tmp.path().join("repo");
16782            std::fs::create_dir(&repo).unwrap();
16783            init_repo(&repo);
16784            let run_git = |args: &[&str]| {
16785                let out = std::process::Command::new("git")
16786                    .args(args)
16787                    .current_dir(&repo)
16788                    .output()
16789                    .unwrap();
16790                assert!(
16791                    out.status.success(),
16792                    "{args:?}: {}",
16793                    String::from_utf8_lossy(&out.stderr)
16794                );
16795            };
16796            let origin = tmp.path().join("origin.git");
16797            run_git(&["init", "--bare", origin.to_str().unwrap()]);
16798            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
16799            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
16800            run_git(&["add", "spec.rs"]);
16801            run_git(&["commit", "-m", "initial spec"]);
16802            run_git(&["push", "origin", "main"]);
16803            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16804            run_git(&["checkout", "-b", "fix/review"]);
16805            std::fs::write(
16806                repo.join("check.rs"),
16807                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
16808            )
16809            .unwrap();
16810            run_git(&["add", "check.rs"]);
16811            run_git(&["commit", "-m", "use spec"]);
16812            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
16813            run_git(&["checkout", "main"]);
16814            std::fs::write(
16815                repo.join("spec.rs"),
16816                "struct AgentSpec { max_turn_secs: u64 }\n",
16817            )
16818            .unwrap();
16819            run_git(&["add", "spec.rs"]);
16820            run_git(&["commit", "-m", "extend spec"]);
16821            run_git(&["push", "origin", "main"]);
16822            run_git(&["checkout", "fix/review"]);
16823
16824            let mut runner = runner_at(RunStatus::Reviewing);
16825            runner.state.repo = repo.clone();
16826            runner.state.base_commit = base;
16827            runner.state.pr_head = Some("fix/review".into());
16828            runner.state.config.graph.review_rounds = max_rounds;
16829            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16830            runner.state.config.graph.sessions = false;
16831            runner.state.config.disk.min_free_bytes = 0;
16832            runner.state.config.verify.e2e =
16833                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
16834            runner.state.candidates = vec![Candidate {
16835                index: 0,
16836                label: 'A',
16837                agent: "fixture".into(),
16838                branch: "fix/review".into(),
16839                worktree: repo.clone(),
16840                summary: String::new(),
16841                stat: String::new(),
16842                files: 1,
16843                commits: 1,
16844                empty: false,
16845                failed: None,
16846                verified_noop: None,
16847                duration_ms: 0,
16848                folded: false,
16849            }];
16850            runner.state.tally = Some(Tally {
16851                first_choice: BTreeMap::new(),
16852                borda: BTreeMap::new(),
16853                winner: 'A',
16854                rankings: 0,
16855                unanimous_initial: true,
16856                deliberated: false,
16857                changed_votes: 0,
16858                unanimous_final: true,
16859                tie_break: None,
16860                judges: 0,
16861                present: 0,
16862                quorum: 0,
16863                met_quorum: true,
16864                uncontested: Some("review-only".into()),
16865            });
16866            let mut last = review_round(false, 0, 1, 1, false, false);
16867            last.head = reviewed.clone();
16868            last.e2e_deferred = true;
16869            last.e2e[0].resource_blocked = true;
16870            last.e2e[0].output_tail = "waiting for the shared build cache".into();
16871            if max_rounds == 1 {
16872                last.e2e.clear();
16873            }
16874            runner.state.reviews = vec![last];
16875            let calls = tmp.path().join("fix-calls");
16876            let mut fixer = spec("repair");
16877            fixer
16878                .env
16879                .insert("FIX_CALLS".into(), calls.display().to_string());
16880            fixer.command = vec!["sh".into(), "-c".into(), r#"
16881                echo fix >> "$FIX_CALLS"
16882                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
16883                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
16884                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
16885                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
16886            "#.into()];
16887            let mut reviewer = spec("reviewer");
16888            reviewer.command = vec![
16889                "sh".into(),
16890                "-c".into(),
16891                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
16892                    .into(),
16893            ];
16894            runner.roles.fixer = Some(vec![fixer]);
16895            runner.roles.reviewers = vec![reviewer];
16896
16897            runner.sync_to_base().await.unwrap();
16898            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
16899            assert_ne!(rebased, reviewed);
16900            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
16901            if reentry {
16902                // Simulate a crash after catch-up evidence was saved but before
16903                // dispatching the fixer. No dedicated pending-repair field exists.
16904                let shell = runner.state.config.shell();
16905                runner
16906                    .stop_unresolved_review("deferred verification", &shell, &repo)
16907                    .await
16908                    .unwrap();
16909                assert!(runner.catchup_repairable());
16910                assert!(!calls.exists());
16911                runner.state = RunState::load(&runner.state.id).unwrap();
16912                assert!(runner.catchup_repairable());
16913            }
16914            runner.review_loop().await.unwrap();
16915            let first = &runner.state.reviews[0];
16916            assert_eq!(first.head, reviewed);
16917            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
16918            assert!(
16919                first.verify_retried,
16920                "a reproduced build failure gets exactly one retry: {:?}",
16921                first.e2e
16922            );
16923            assert!(first.e2e[0].output_tail.contains("E0063"));
16924            assert!(!runner.catchup_repairable());
16925            if max_rounds == 2 {
16926                assert_eq!(
16927                    runner.state.status,
16928                    RunStatus::Gating,
16929                    "{}",
16930                    crate::report::run(&runner.state)
16931                );
16932                assert_eq!(runner.state.reviews.len(), 2);
16933                assert!(first.fix.as_ref().unwrap().committed);
16934                assert!(runner.state.reviews[1].clean);
16935                assert_eq!(
16936                    runner
16937                        .state
16938                        .reviews
16939                        .iter()
16940                        .filter(|r| r.fix.is_some())
16941                        .count(),
16942                    1
16943                );
16944                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
16945                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
16946            } else {
16947                assert_eq!(runner.state.status, RunStatus::Blocked);
16948                assert!(first.fix.is_none());
16949                assert!(!calls.exists());
16950                let recovery = crate::recovery::assess(
16951                    "prior",
16952                    Some(&runner.state),
16953                    &runner.landing_base(),
16954                    crate::recovery::Relation::Diverged,
16955                );
16956                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
16957                let reasons = recovery.reasons.join("\n");
16958                assert!(
16959                    reasons.contains(&reviewed) && reasons.contains(&rebased),
16960                    "{reasons}"
16961                );
16962                assert!(
16963                    reasons.contains("base sync to")
16964                        && reasons.contains("additional review is required"),
16965                    "{reasons}"
16966                );
16967            }
16968        }
16969    }
16970
16971    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
16972    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
16973    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
16974    /// own top-of-function fast path (`review_conclusion`) correctly reads
16975    /// this shape as `None` rather than guessing `Blocked`, and the loop's
16976    /// own `for` range is empty once the round budget is spent, so
16977    /// `review_loop` must retry the check itself rather than silently doing
16978    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
16979    /// above exercises directly, but through `review_loop`'s own entry point
16980    /// this time, proving the wiring between the two rather than just the
16981    /// retry logic in isolation.
16982    #[tokio::test]
16983    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16984        crate::run::pin_test_home();
16985        let home = crate::run::home();
16986
16987        let tmp = tempfile::tempdir().expect("tempdir");
16988        let repo = tmp.path().join("repo");
16989        std::fs::create_dir_all(&repo).unwrap();
16990        init_repo(&repo);
16991        let head = crate::git::rev_parse(&repo, "HEAD")
16992            .await
16993            .expect("rev-parse");
16994        let cache_dir = tmp.path().join("target");
16995
16996        let mut config = Config::default();
16997        config.verify.e2e = vec![format!(
16998            "CARGO_TARGET_DIR='{}' test -f README.md",
16999            cache_dir.display()
17000        )];
17001        config.graph.review_rounds = 1;
17002        config.graph.timeout_verify = Some(2);
17003        config.graph.timeout_resource_wait = 2;
17004
17005        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17006        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17007            .expect("no io error acquiring directly")
17008        {
17009            crate::cache::AcquireOutcome::Acquired(g) => g,
17010            crate::cache::AcquireOutcome::Busy(b) => {
17011                panic!("expected the direct acquire to win the lease first: {b:?}")
17012            }
17013        };
17014
17015        let mut state = RunState::new(
17016            repo.clone(),
17017            "main".to_owned(),
17018            head.clone(),
17019            "task".to_owned(),
17020            config,
17021        );
17022        state.candidates = vec![Candidate {
17023            index: 0,
17024            label: 'A',
17025            agent: "alpha".to_owned(),
17026            branch: "does-not-exist".to_owned(),
17027            worktree: repo.clone(),
17028            summary: String::new(),
17029            stat: String::new(),
17030            files: 0,
17031            commits: 0,
17032            empty: false,
17033            failed: None,
17034            verified_noop: None,
17035            duration_ms: 0,
17036            folded: false,
17037        }];
17038        state.tally = Some(Tally {
17039            first_choice: BTreeMap::from([('A', 1)]),
17040            borda: BTreeMap::new(),
17041            winner: 'A',
17042            rankings: 1,
17043            unanimous_initial: true,
17044            deliberated: false,
17045            changed_votes: 0,
17046            unanimous_final: true,
17047            tie_break: None,
17048            judges: 0,
17049            present: 0,
17050            quorum: 0,
17051            met_quorum: true,
17052            uncontested: Some("only candidate A produced a change".to_owned()),
17053        });
17054        // The exact shape a prior process's `stop_reviewing` would have left
17055        // on disk: the round budget's last round, a real attempt already
17056        // made and already resource-blocked.
17057        state.reviews = vec![ReviewRound {
17058            round: 1,
17059            head: head.clone(),
17060            verified_head: Some(head.clone()),
17061            verified_at: Some(jiff::Timestamp::now()),
17062            reviews: Vec::new(),
17063            e2e: vec![CommandOutcome {
17064                command: format!(
17065                    "CARGO_TARGET_DIR='{}' test -f README.md",
17066                    cache_dir.display()
17067                ),
17068                code: None,
17069                output_tail: "waiting for the shared build cache".to_owned(),
17070                duration_ms: 0,
17071                resource_blocked: true,
17072            }],
17073            fix: None,
17074            blocking: 1,
17075            answered: 1,
17076            expected: 1,
17077            clean: false,
17078            verify_retried: false,
17079            e2e_deferred: false,
17080            e2e_defer_reason: None,
17081            progressed: false,
17082            vote_split: false,
17083            reconsideration: Vec::new(),
17084            verdict: None,
17085        }];
17086
17087        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17088        let mut runner = Runner {
17089            state,
17090            roles: ResolvedRoles {
17091                implementers: Vec::new(),
17092                judges: Vec::new(),
17093                reviewers: Vec::new(),
17094                fixer: None,
17095                conductor: conductor(),
17096                implementer_roster: Vec::new(),
17097                judge_roster: Vec::new(),
17098                reviewer_roster: Vec::new(),
17099            },
17100            sem: Arc::new(Semaphore::new(1)),
17101            pause: Pause::new(),
17102            interrupt: Pause::new(),
17103        };
17104
17105        // The lease is still held throughout, so this reentry's own retry is
17106        // also contended — proving `review_loop` actually tried again (not
17107        // that it happened to succeed) is what the timestamp comparison
17108        // below is for.
17109        runner.review_loop().await.expect("review_loop");
17110
17111        assert_eq!(
17112            runner.state.reviews.len(),
17113            1,
17114            "no new round was started on top of the unresolved one: {:?}",
17115            runner.state.reviews
17116        );
17117        let last = &runner.state.reviews[0];
17118        assert_eq!(
17119            last.e2e_status(),
17120            E2eStatus::ResourceBlocked,
17121            "still contended: {last:?}"
17122        );
17123        assert!(
17124            last.verified_at.expect("still known") > first_attempt_at,
17125            "review_loop must have actually retried the check, not left it exactly as found"
17126        );
17127        assert_ne!(
17128            runner.state.status,
17129            RunStatus::Blocked,
17130            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17131            runner.state.status
17132        );
17133
17134        held.release();
17135    }
17136
17137    #[tokio::test]
17138    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17139        crate::run::pin_test_home();
17140        for status in [RunStatus::Blocked, RunStatus::Gating] {
17141            let mut runner = runner_at(status);
17142            runner.state.config.graph.land = true;
17143            runner.state.gate_ran = true;
17144            runner.state.merge = Some(MergeOutcome {
17145                mode: MergeMode::Pr,
17146                ok: false,
17147                detail: "the fixer produced no commit".into(),
17148                empty: false,
17149            });
17150            let url = "https://example.invalid/o/r/pull/1";
17151            runner.state.pr = Some(crate::run::PrRecord {
17152                url: url.into(),
17153                number: 1,
17154                state: "open".into(),
17155                checks: "red".into(),
17156                round: 1,
17157                rounds: 1,
17158                red_at_merge: Vec::new(),
17159            });
17160            runner.state.save().unwrap();
17161            // The nonexistent repository makes observation fail before a forge
17162            // command can run. We still must reach land, rather than silently
17163            // return from the graph with its old gate and merge records.
17164            runner.execute_graph().await.unwrap();
17165            assert_eq!(runner.state.status, RunStatus::Blocked);
17166            assert!(runner.state.land_sync_pending);
17167            assert!(runner.state.events.iter().any(|event|
17168                event.node == "land" && event.message == format!("watching {url}")));
17169            assert_eq!(
17170                runner.state.merge.as_ref().unwrap().detail,
17171                "the fixer produced no commit"
17172            );
17173        }
17174    }
17175
17176    #[tokio::test]
17177    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17178        crate::run::pin_test_home();
17179        let tmp = tempfile::tempdir().expect("tempdir");
17180        let repo = tmp.path().join("repo");
17181        std::fs::create_dir_all(&repo).unwrap();
17182        init_repo(&repo);
17183
17184        let mut config = Config::default();
17185        config.merge.mode = MergeMode::Pr;
17186        config.graph.land = true;
17187        config.graph.land_approval = false;
17188
17189        let mut state = RunState::new(
17190            repo.clone(),
17191            "main".to_owned(),
17192            "deadbeef".to_owned(),
17193            "task".to_owned(),
17194            config,
17195        );
17196        state.candidates = vec![Candidate {
17197            index: 0,
17198            label: 'A',
17199            agent: "alpha".to_owned(),
17200            branch: "does-not-exist".to_owned(),
17201            worktree: repo.clone(),
17202            summary: String::new(),
17203            stat: String::new(),
17204            files: 0,
17205            commits: 0,
17206            empty: false,
17207            failed: None,
17208            verified_noop: None,
17209            duration_ms: 0,
17210            folded: false,
17211        }];
17212        state.tally = Some(Tally {
17213            first_choice: BTreeMap::from([('A', 1)]),
17214            borda: BTreeMap::new(),
17215            winner: 'A',
17216            rankings: 1,
17217            unanimous_initial: true,
17218            deliberated: false,
17219            changed_votes: 0,
17220            unanimous_final: true,
17221            tie_break: None,
17222            judges: 0,
17223            present: 0,
17224            quorum: 0,
17225            met_quorum: true,
17226            uncontested: Some("only candidate A produced a change".to_owned()),
17227        });
17228        state.reviews = vec![ReviewRound {
17229            round: 1,
17230            head: "deadbeef".to_owned(),
17231            verified_head: None,
17232            verified_at: None,
17233            reviews: Vec::new(),
17234            e2e: Vec::new(),
17235            fix: None,
17236            blocking: 0,
17237            answered: 0,
17238            expected: 0,
17239            clean: true,
17240            verify_retried: false,
17241            e2e_deferred: false,
17242            e2e_defer_reason: None,
17243            progressed: false,
17244            vote_split: false,
17245            reconsideration: Vec::new(),
17246            verdict: None,
17247        }];
17248        state.gate = vec![CommandOutcome {
17249            command: "test".to_owned(),
17250            code: Some(0),
17251            output_tail: String::new(),
17252            duration_ms: 0,
17253            resource_blocked: false,
17254        }];
17255        state.gate_ran = true;
17256        // A first pass through `merge` already pushed and opened this pull
17257        // request; `status` is `Landing` because a previous call into `land`
17258        // parked or was interrupted before it reached a terminal outcome.
17259        state.status = RunStatus::Landing;
17260        state.merge = Some(MergeOutcome {
17261            mode: MergeMode::Pr,
17262            ok: true,
17263            detail: "https://example.invalid/x/y/pull/1".to_owned(),
17264            empty: false,
17265        });
17266
17267        // The Landing-resume shortcut calls `run_land` directly rather than
17268        // through `merge`, which is exactly the call site that used to skip
17269        // `settle_questions` - see the fixture below.
17270        ask_test_home();
17271        // A real resume loads a persisted run; driver ownership is checked
17272        // against that record before the Landing shortcut is entered.
17273        state.save().expect("save landing run");
17274        let store = ask::Questions::open();
17275        let q = ask_open_question(&store, &state.id);
17276
17277        let mut runner = Runner {
17278            state,
17279            roles: ResolvedRoles {
17280                implementers: Vec::new(),
17281                judges: Vec::new(),
17282                reviewers: Vec::new(),
17283                fixer: None,
17284                conductor: conductor(),
17285                implementer_roster: Vec::new(),
17286                judge_roster: Vec::new(),
17287                reviewer_roster: Vec::new(),
17288            },
17289            sem: Arc::new(Semaphore::new(1)),
17290            pause: Pause::new(),
17291            interrupt: Pause::new(),
17292        };
17293
17294        // `execute`, not `merge` directly: the Landing-resume shortcut lives
17295        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
17296        // exactly because `review_loop` would otherwise clobber the marker
17297        // first.
17298        runner.execute().await.expect("execute");
17299
17300        assert_eq!(
17301            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17302            Some("https://example.invalid/x/y/pull/1"),
17303            "reentry must not push again or open a second pull request over the \
17304             one `land` is already watching"
17305        );
17306        assert_ne!(
17307            runner.state.status,
17308            RunStatus::Landing,
17309            "land could not actually reach the fake pull request, so it must \
17310             have given up rather than left the run silently parked forever"
17311        );
17312        // `land` could not reach the fake pull request, so it gave up into
17313        // `Blocked` - still resumable, so the question must not have been
17314        // swept just because this branch now also calls `settle_questions`.
17315        assert_eq!(runner.state.status, RunStatus::Blocked);
17316        assert!(
17317            store.get(&q.id).unwrap().status.open(),
17318            "Blocked is still alive; settle_questions must have been a no-op here"
17319        );
17320    }
17321
17322    fn state_with_round(round: ReviewRound) -> RunState {
17323        let mut s = RunState::new(
17324            PathBuf::from("/repo"),
17325            "main".to_owned(),
17326            "abc1234".to_owned(),
17327            "add retries".to_owned(),
17328            Config::default(),
17329        );
17330        s.reviews = vec![round];
17331        s
17332    }
17333
17334    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17335        crate::verdict::Finding {
17336            id: id.to_owned(),
17337            severity,
17338            file: None,
17339            line: None,
17340            title: title.to_owned(),
17341            detail: String::new(),
17342        }
17343    }
17344
17345    #[test]
17346    fn pr_body_names_open_findings_and_declined_ones() {
17347        let round = ReviewRound {
17348            round: 2,
17349            head: "deadbee".to_owned(),
17350            verified_head: None,
17351            verified_at: None,
17352            reviews: vec![ReviewRecord {
17353                attempts: 0,
17354                reviewer: 1,
17355                agent: "alpha".to_owned(),
17356                summary: String::new(),
17357                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17358                vote: None,
17359                failed: None,
17360                duration_ms: 0,
17361            }],
17362            e2e: vec![CommandOutcome {
17363                command: "cargo test".to_owned(),
17364                code: Some(0),
17365                output_tail: String::new(),
17366                duration_ms: 0,
17367                resource_blocked: false,
17368            }],
17369            verify_retried: false,
17370            e2e_deferred: false,
17371            e2e_defer_reason: None,
17372            fix: Some(FixRecord {
17373                agent: "alpha".to_owned(),
17374                addressed: Vec::new(),
17375                rejected: vec![crate::verdict::Rejection {
17376                    id: "R1-1-1".to_owned(),
17377                    why: "not reachable from any caller".to_owned(),
17378                }],
17379                notes: String::new(),
17380                committed: true,
17381                failed: None,
17382                duration_ms: 0,
17383                continuation: None,
17384            }),
17385            blocking: 0,
17386            answered: 1,
17387            expected: 1,
17388            clean: false,
17389            progressed: true,
17390            vote_split: false,
17391            reconsideration: Vec::new(),
17392            verdict: None,
17393        };
17394        let state = state_with_round(round);
17395        let body = pr_message(&state, 'A').body;
17396
17397        assert!(body.contains("add retries"), "the task must still be there");
17398        assert!(body.contains("R2-1-1"), "{body}");
17399        assert!(!body.contains("unused import"), "{body}");
17400        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17401        assert!(
17402            !body.contains("not reachable from any caller"),
17403            "the reason it was declined: {body}"
17404        );
17405    }
17406
17407    #[test]
17408    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17409        let round = ReviewRound {
17410            round: 1,
17411            head: "deadbee".to_owned(),
17412            verified_head: None,
17413            verified_at: None,
17414            reviews: vec![ReviewRecord {
17415                attempts: 0,
17416                reviewer: 1,
17417                agent: "alpha".to_owned(),
17418                summary: String::new(),
17419                findings: Vec::new(),
17420                vote: None,
17421                failed: None,
17422                duration_ms: 0,
17423            }],
17424            e2e: Vec::new(),
17425            verify_retried: false,
17426            e2e_deferred: false,
17427            e2e_defer_reason: None,
17428            fix: None,
17429            blocking: 0,
17430            answered: 1,
17431            expected: 1,
17432            clean: true,
17433            progressed: false,
17434            vote_split: false,
17435            reconsideration: Vec::new(),
17436            verdict: None,
17437        };
17438        let state = state_with_round(round);
17439        let body = pr_message(&state, 'A').body;
17440        assert!(!body.contains("Open review findings"), "{body}");
17441        assert!(!body.contains("Declined"), "{body}");
17442    }
17443
17444    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17445        let mut state = RunState::new(
17446            PathBuf::from("/repo"),
17447            "main".to_owned(),
17448            "abc1234".to_owned(),
17449            instruction.to_owned(),
17450            Config::default(),
17451        );
17452        state.candidates.push(Candidate {
17453            index: 0,
17454            label: 'A',
17455            agent: "alpha".to_owned(),
17456            branch: "magi/x/A".to_owned(),
17457            worktree: PathBuf::from("/wt"),
17458            summary: summary.to_owned(),
17459            stat: String::new(),
17460            files: 1,
17461            commits: 1,
17462            empty: false,
17463            failed: None,
17464            verified_noop: None,
17465            folded: false,
17466            duration_ms: 0,
17467        });
17468        state
17469    }
17470
17471    fn review_state(subjects: &[&str]) -> RunState {
17472        let mut state = state_with_summary(
17473            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17474            "",
17475        );
17476        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17477        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17478        state
17479    }
17480
17481    /// A branch rebased onto a main that moved past the recorded
17482    /// `base_commit` is titled from its own first commit, never main's.
17483    #[tokio::test]
17484    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17485        ask_test_home();
17486        let tmp = tempfile::tempdir().unwrap();
17487        let repo = tmp.path().join("repo");
17488        std::fs::create_dir_all(&repo).unwrap();
17489        init_repo(&repo);
17490        let origin = tmp.path().join("origin.git");
17491        let g = |dir: &Path, args: &[&str]| {
17492            let out = std::process::Command::new("git")
17493                .args(args)
17494                .current_dir(dir)
17495                .quiet()
17496                .output()
17497                .expect("spawn git");
17498            assert!(
17499                out.status.success(),
17500                "git {args:?}: {}",
17501                String::from_utf8_lossy(&out.stderr)
17502            );
17503        };
17504        g(
17505            tmp.path(),
17506            &[
17507                "clone",
17508                "--bare",
17509                "-q",
17510                repo.to_str().unwrap(),
17511                origin.to_str().unwrap(),
17512            ],
17513        );
17514        g(
17515            &repo,
17516            &["remote", "add", "origin", origin.to_str().unwrap()],
17517        );
17518        let c1 = git::rev_parse(&repo, "main").await.unwrap();
17519
17520        // Main moves on; the branch is built on top of the new main.
17521        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17522        g(&repo, &["add", "-A"]);
17523        g(
17524            &repo,
17525            &["commit", "-q", "-m", "chore(deps): update a crate"],
17526        );
17527        g(&repo, &["push", "-q", "origin", "main"]);
17528        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17529        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17530        g(&repo, &["add", "-A"]);
17531        g(
17532            &repo,
17533            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17534        );
17535        g(&repo, &["checkout", "-q", "main"]);
17536
17537        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17538        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17539        // Without a readable tracking ref the recorded base's merge base is used.
17540        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17541        assert_eq!(fallback, c1);
17542
17543        let mut state = review_state(&[
17544            "chore(deps): update a crate",
17545            "fix(daemon): apply a chosen action",
17546        ]);
17547        state.repo = repo.clone();
17548        state.base_branch = "main".to_owned();
17549        state.base_commit = c1;
17550        refresh_reviewed_commits(&mut state, "feat/own").await;
17551        assert_eq!(
17552            state.reviewed_commits,
17553            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17554        );
17555        assert_eq!(
17556            review_title(&state).as_deref(),
17557            Some("fix(daemon): apply a chosen action")
17558        );
17559        assert_eq!(
17560            leaked_subjects(&state, "feat/own").await,
17561            Some(vec!["chore(deps): update a crate".to_owned()])
17562        );
17563        // A stale tracking ref is still used when the fetch fails, but the
17564        // leak list is withheld.
17565        state.config.merge.remote = "nowhere".to_owned();
17566        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17567    }
17568
17569    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
17570    /// with `origin/main` one commit behind it, and a summarizing agent that
17571    /// prints whatever `reply` holds.
17572    struct SummaryFixture {
17573        _tmp: tempfile::TempDir,
17574        repo: PathBuf,
17575        reply: PathBuf,
17576        runner: Runner,
17577    }
17578
17579    fn summary_fixture() -> SummaryFixture {
17580        ask_test_home();
17581        let tmp = tempfile::tempdir().unwrap();
17582        let repo = tmp.path().join("repo");
17583        std::fs::create_dir_all(&repo).unwrap();
17584        init_repo(&repo);
17585        let origin = tmp.path().join("origin.git");
17586        let g = |args: &[&str]| {
17587            let out = std::process::Command::new("git")
17588                .args(args)
17589                .current_dir(&repo)
17590                .quiet()
17591                .output()
17592                .expect("spawn git");
17593            assert!(
17594                out.status.success(),
17595                "git {args:?}: {}",
17596                String::from_utf8_lossy(&out.stderr)
17597            );
17598        };
17599        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17600        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17601        g(&["push", "-q", "origin", "main"]);
17602        g(&["checkout", "-q", "-b", "feat/own"]);
17603        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17604        g(&["add", "-A"]);
17605        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17606        let reply = tmp.path().join("reply.txt");
17607        std::fs::write(
17608            &reply,
17609            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17610        )
17611        .unwrap();
17612        let mut runner = runner_at(RunStatus::Gating);
17613        let mut state = review_state(&["feat(own): add the own file"]);
17614        state.repo = repo.clone();
17615        state.base_branch = "main".to_owned();
17616        state.candidates[0].branch = "feat/own".to_owned();
17617        state.candidates[0].worktree = repo.clone();
17618        let mut author = spec("alpha");
17619        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17620        author
17621            .env
17622            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17623        state.config.agents = vec![author];
17624        runner.state = state;
17625        SummaryFixture {
17626            _tmp: tmp,
17627            repo,
17628            reply,
17629            runner,
17630        }
17631    }
17632
17633    async fn posted(f: &mut SummaryFixture) -> PrMessage {
17634        let winner = f.runner.state.candidates[0].clone();
17635        f.runner
17636            .guarded_pr_message(&winner, None, true)
17637            .await
17638            .unwrap()
17639            .expect("an English description never parks")
17640    }
17641
17642    fn clean_round_with_minor_findings() -> ReviewRound {
17643        let mut round = passing_round(2, None);
17644        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17645        round.reviews = vec![ReviewRecord {
17646            attempts: 0,
17647            reviewer: 1,
17648            agent: "alpha".into(),
17649            summary: "内部レビュー".into(),
17650            findings: vec![
17651                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17652                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17653            ],
17654            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17655            failed: None,
17656            duration_ms: 0,
17657        }];
17658        round
17659    }
17660
17661    #[tokio::test]
17662    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17663        for review_only in [false, true] {
17664            let mut f = summary_fixture();
17665            if !review_only {
17666                f.runner.state.candidates[0].agent = "alpha".into();
17667            }
17668            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17669            assert!(f.runner.state.reviews[0].clean);
17670            assert!(f.runner.state.reviews[0].fix.is_none());
17671            assert!(f.runner.state.open_findings().is_empty());
17672            assert_eq!(
17673                crate::pr_outcome::required_finding_ids(&f.runner.state),
17674                ["R2-1-1", "R2-1-2"]
17675            );
17676            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17677            let evidence = crate::pr_outcome::summary_evidence(
17678                &f.runner.state,
17679                crate::pr_outcome::FinalHead::At(&head),
17680            );
17681            assert!(
17682                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17683            );
17684            // The ordinary reply omits both IDs: no cached description is adopted.
17685            let omitted = posted(&mut f).await;
17686            assert!(omitted.body.contains("No description"));
17687            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17688            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17689            assert!(f.runner.state.change_summary.is_none());
17690            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17691            let described = posted(&mut f).await;
17692            assert!(described.body.contains("improve documentation for callers"));
17693            assert!(described.body.contains("remaining edge case"));
17694            assert!(described.body.contains("## Open review findings"));
17695            assert!(!described.body.contains("未修正"));
17696        }
17697    }
17698
17699    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17700        let mut f = summary_fixture();
17701        if !review_only {
17702            f.runner.state.candidates[0].agent = "alpha".into();
17703        }
17704        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17705        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17706        fixed
17707            .fix
17708            .as_mut()
17709            .unwrap()
17710            .rejected
17711            .push(crate::verdict::Rejection {
17712                id: "R1-1-1".into(),
17713                why: "呼び出されないコードのため辞退".into(),
17714            });
17715        let mut clean = clean_round_with_minor_findings();
17716        clean.verified_head = Some(head.clone());
17717        f.runner.state.reviews = vec![fixed, clean];
17718        ok_gate(&mut f.runner.state, Some(&head));
17719        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17720        f.runner.state.config.agents.push(scripted("beta", reply));
17721        f.runner.state.config.roles.language_judge =
17722            Some(crate::config::AgentChoice::One("beta".into()));
17723        f
17724    }
17725
17726    #[tokio::test]
17727    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17728        for review_only in [false, true] {
17729            let mut f = rewrite_fixture_with_findings(review_only,
17730                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17731            let m = posted(&mut f).await;
17732            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17733            let expected =
17734                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17735            assert!(m.body.contains(&expected));
17736            assert!(m.body.contains("## Review rounds"));
17737            assert!(
17738                m.body
17739                    .contains("Gate (`verify.gate`): passed on the final commit")
17740            );
17741            assert!(
17742                m.body
17743                    .contains("Tests (`verify.e2e`): passed on the final commit")
17744            );
17745            let c = f.runner.state.change_summary.as_ref().unwrap();
17746            assert!(
17747                m.body
17748                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17749            );
17750            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17751            assert!(crate::pr_outcome::covers_required_findings(
17752                &f.runner.state,
17753                &m.body
17754            ));
17755            assert!(m.body.contains("documentation remains incomplete"));
17756            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17757            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17758            assert!(f.runner.state.github_text.is_none());
17759        }
17760    }
17761
17762    #[tokio::test]
17763    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17764        for review_only in [false, true] {
17765            for reply in [
17766                "TITLE: fix: public description\nA short English description.",
17767                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17768            ] {
17769                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17770                let winner = f.runner.state.candidates[0].clone();
17771                let posted = f
17772                    .runner
17773                    .guarded_pr_message(&winner, None, true)
17774                    .await
17775                    .unwrap();
17776                assert!(posted.is_none());
17777                assert!(f.runner.state.parked);
17778                let withheld = f.runner.state.github_text.as_ref().unwrap();
17779                assert!(
17780                    withheld.rewrite.is_none(),
17781                    "an incomplete rewrite must not be selectable"
17782                );
17783                let q = ask::Questions::open()
17784                    .get(withheld.question.as_ref().unwrap())
17785                    .unwrap();
17786                assert!(
17787                    !q.choices
17788                        .contains(&crate::github_text::USE_REWRITE.to_owned())
17789                );
17790                assert!(
17791                    q.detail.contains("R1-1-1")
17792                        && q.detail.contains("R2-1-1")
17793                        && q.detail.contains("R2-1-2")
17794                );
17795            }
17796        }
17797    }
17798
17799    #[tokio::test]
17800    async fn japanese_reports_stay_local_in_both_public_run_modes() {
17801        for review_only in [false, true] {
17802            let mut f = summary_fixture();
17803            if !review_only {
17804                f.runner.state.candidates[0].agent = "alpha".into();
17805            }
17806            f.runner.state.instruction = "内部タスクの説明".into();
17807            f.runner.state.candidates[0].summary = "内部の候補報告".into();
17808            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17809            f.runner.state.reviews = vec![
17810                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17811                round_with_notes(2, Some("二回目の内部修正報告です。")),
17812                passing_round(3, Some(&head)),
17813            ];
17814            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17815            ok_gate(&mut f.runner.state, Some(&head));
17816            let captured = f.repo.join("public-prompt.txt");
17817            let author = &mut f.runner.state.config.agents[0];
17818            author.command = vec![
17819                "sh".into(),
17820                "-c".into(),
17821                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17822            ];
17823            author.env.insert(
17824                "PROMPT_FILE".into(),
17825                captured.to_string_lossy().into_owned(),
17826            );
17827            let m = posted(&mut f).await;
17828            assert!(m.body.contains("Adds `own.txt`"));
17829            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17830            assert!(m.body.contains("R1-1-1"));
17831            assert!(
17832                m.body
17833                    .contains("Gate (`verify.gate`): passed on the final commit")
17834            );
17835            assert!(
17836                m.body
17837                    .contains("Tests (`verify.e2e`): passed on the final commit")
17838            );
17839            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17840            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17841            let prompt = std::fs::read_to_string(captured).unwrap();
17842            assert!(prompt.contains(&format!(
17843                "git diff {} {head}",
17844                f.runner.state.change_summary.as_ref().unwrap().base
17845            )));
17846            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17847            assert_eq!(
17848                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17849                "二回目の内部修正報告です。"
17850            );
17851        }
17852    }
17853
17854    #[tokio::test]
17855    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17856        for review_only in [false, true] {
17857            let mut f = summary_fixture();
17858            if !review_only {
17859                f.runner.state.candidates[0].agent = "alpha".into();
17860            }
17861            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17862            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17863            std::fs::write(&f.reply, "no usable description").unwrap();
17864            let failed = posted(&mut f).await;
17865            assert!(
17866                failed
17867                    .body
17868                    .contains("No description of what this change does could be generated")
17869            );
17870            assert!(failed.body.is_ascii());
17871            assert!(f.runner.state.github_text.is_none());
17872            assert!(f.runner.state.change_summary.is_none());
17873            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17874            let evidence = crate::pr_outcome::summary_evidence(
17875                &f.runner.state,
17876                crate::pr_outcome::FinalHead::At(&head),
17877            );
17878            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17879            assert!(
17880                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17881                    .exists()
17882            );
17883            f.runner.state =
17884                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17885            std::fs::write(
17886                &f.reply,
17887                "TITLE: feat(own): add the own file\nAdds the committed own file.",
17888            )
17889            .unwrap();
17890            let retried = posted(&mut f).await;
17891            assert!(retried.body.contains("Adds the committed own file."));
17892            assert_eq!(
17893                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17894                "日本語の修正報告"
17895            );
17896        }
17897    }
17898
17899    #[tokio::test]
17900    async fn same_head_verification_and_findings_invalidate_public_prose() {
17901        let mut f = summary_fixture();
17902        posted(&mut f).await;
17903        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17904        let original = f
17905            .runner
17906            .state
17907            .change_summary
17908            .as_ref()
17909            .unwrap()
17910            .evidence
17911            .clone();
17912        ok_gate(&mut f.runner.state, Some(&head));
17913        std::fs::write(
17914            &f.reply,
17915            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17916        )
17917        .unwrap();
17918        let verified = posted(&mut f).await;
17919        assert!(verified.body.contains("now verified"));
17920        assert_ne!(
17921            f.runner.state.change_summary.as_ref().unwrap().evidence,
17922            original
17923        );
17924        let mut round = round_with_notes(1, Some("内部報告"));
17925        round.reviews.push(ReviewRecord {
17926            attempts: 0,
17927            reviewer: 1,
17928            agent: "alpha".into(),
17929            summary: "内部レビュー".into(),
17930            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17931            vote: None,
17932            failed: None,
17933            duration_ms: 0,
17934        });
17935        round
17936            .fix
17937            .as_mut()
17938            .unwrap()
17939            .rejected
17940            .push(crate::verdict::Rejection {
17941                id: "R1-1-1".into(),
17942                why: "呼び出し元から到達できない".into(),
17943            });
17944        f.runner.state.reviews = vec![round];
17945        // A description omitting the required ID is unusable, even with a title.
17946        let omitted = posted(&mut f).await;
17947        assert!(omitted.body.contains("No description"));
17948        assert!(omitted.body.contains("R1-1-1"));
17949        assert!(!omitted.body.contains("到達"));
17950        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17951        let described = posted(&mut f).await;
17952        assert!(described.body.contains("unreachable from callers"));
17953        assert!(described.body.contains("R1-1-1"));
17954        assert!(!described.body.contains("到達"));
17955        let evidence = f
17956            .runner
17957            .state
17958            .change_summary
17959            .as_ref()
17960            .unwrap()
17961            .evidence
17962            .clone();
17963        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17964        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17965        assert!(posted(&mut f).await.body.contains("rationale updated"));
17966        assert_ne!(
17967            f.runner.state.change_summary.as_ref().unwrap().evidence,
17968            evidence
17969        );
17970        // Old serialized summaries have no evidence fingerprint and must regenerate.
17971        f.runner
17972            .state
17973            .change_summary
17974            .as_mut()
17975            .unwrap()
17976            .evidence
17977            .clear();
17978        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17979        assert!(posted(&mut f).await.body.contains("current specification"));
17980    }
17981
17982    #[tokio::test]
17983    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17984        let mut f = summary_fixture();
17985        let m = posted(&mut f).await;
17986        assert_eq!(m.title, "feat(own): add the own file");
17987        assert!(
17988            m.body
17989                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17990            "{}",
17991            m.body
17992        );
17993        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17994        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17995        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17996        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17997        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17998        let kept = f.runner.state.change_summary.as_ref().unwrap();
17999        assert_eq!(
18000            (kept.head.as_str(), kept.base.as_str()),
18001            (head.as_str(), base.as_str())
18002        );
18003        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
18004    }
18005
18006    #[tokio::test]
18007    async fn review_only_summary_receives_the_configured_github_language() {
18008        let mut f = summary_fixture();
18009        f.runner.state.config.graph.github_language = "ja".into();
18010        let captured = f.repo.join("summary-prompt.txt");
18011        let author = &mut f.runner.state.config.agents[0];
18012        author.command = vec![
18013            "sh".into(),
18014            "-c".into(),
18015            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18016        ];
18017        author.env.insert(
18018            "PROMPT_FILE".into(),
18019            captured.to_string_lossy().into_owned(),
18020        );
18021        std::fs::write(
18022            &f.reply,
18023            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18024        )
18025        .unwrap();
18026        let m = posted(&mut f).await;
18027        let prompt = std::fs::read_to_string(captured).unwrap();
18028        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18029        assert!(!prompt.contains("in English"));
18030        assert_eq!(m.title, "feat(own): ファイルを追加");
18031        assert!(m.body.contains("own.txt を追加します。"));
18032    }
18033
18034    #[tokio::test]
18035    async fn review_fixes_come_after_the_whole_change_summary() {
18036        let mut f = summary_fixture();
18037        f.runner.state.reviews = vec![
18038            round_with_notes(1, Some("handled the PATH case")),
18039            round_with_notes(2, None),
18040        ];
18041        let m = posted(&mut f).await;
18042        let summary = m.body.find("## Summary").expect("summary");
18043        let fixes = m.body.find("## Review rounds").expect("review rounds");
18044        assert!(summary < fixes, "{}", m.body);
18045        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18046        assert!(m.body.contains("## Final verification"), "{}", m.body);
18047    }
18048
18049    #[tokio::test]
18050    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18051        let mut f = summary_fixture();
18052        let first = posted(&mut f).await;
18053        let first_head = f.runner.state.change_summary.clone().unwrap().head;
18054        // The same head reuses the record: a different reply is not read.
18055        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18056        let again = posted(&mut f).await;
18057        assert_eq!((again.title, again.body), (first.title, first.body));
18058
18059        let g = |args: &[&str]| {
18060            let out = std::process::Command::new("git")
18061                .args(args)
18062                .current_dir(&f.repo)
18063                .quiet()
18064                .output()
18065                .expect("spawn git");
18066            assert!(out.status.success(), "git {args:?}");
18067        };
18068        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18069        g(&["add", "-A"]);
18070        g(&["commit", "-q", "-m", "fix(own): handle more"]);
18071        let second = posted(&mut f).await;
18072        let kept = f.runner.state.change_summary.clone().unwrap();
18073        assert_ne!(kept.head, first_head);
18074        assert!(second.body.contains("Not read yet."), "{}", second.body);
18075        assert!(!second.body.contains("so the first summary is read"));
18076        assert_eq!(second.title, "feat(own): other");
18077    }
18078
18079    #[tokio::test]
18080    async fn an_unavailable_summary_is_stated_and_never_invented() {
18081        let mut f = summary_fixture();
18082        std::fs::write(&f.reply, "no title line here").unwrap();
18083        let m = posted(&mut f).await;
18084        assert!(
18085            m.body
18086                .contains("No description of what this change does could be generated"),
18087            "{}",
18088            m.body
18089        );
18090        assert!(m.body.contains("internal reports remain in local records"));
18091        assert!(f.runner.state.change_summary.is_none());
18092        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18093        // A preview asks nobody and claims nothing.
18094        let winner = f.runner.state.candidates[0].clone();
18095        let preview = f
18096            .runner
18097            .guarded_pr_message(&winner, None, false)
18098            .await
18099            .unwrap()
18100            .unwrap();
18101        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18102    }
18103
18104    #[test]
18105    fn pr_message_review_single_commit_uses_its_subject() {
18106        let state = review_state(&["feat(nats): per-role user"]);
18107        let m = pr_message(&state, 'A');
18108        assert_eq!(m.title, "feat(nats): per-role user");
18109        assert!(!m.body.contains("Review the work already"), "{}", m.body);
18110        assert!(m.body.contains("## Commits under review"), "{}", m.body);
18111    }
18112
18113    #[test]
18114    fn pr_message_review_multi_commit_takes_the_oldest() {
18115        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18116        let m = pr_message(&state, 'A');
18117        assert_eq!(m.title, "feat: the change");
18118        for s in ["feat: the change", "fix: typo", "fix: again"] {
18119            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18120        }
18121    }
18122
18123    #[test]
18124    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18125        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18126            let mut state = review_state(&[first, "fix: later fixup"]);
18127            state.candidates[0].branch = "機能/ログイン".to_owned();
18128            let m = pr_message(&state, 'A');
18129            assert!(
18130                m.title.starts_with("chore: land candidate A of run"),
18131                "{}",
18132                m.title
18133            );
18134        }
18135    }
18136
18137    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18138        BranchFacts {
18139            commits: commits
18140                .iter()
18141                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18142                .collect(),
18143            stat: stat.to_owned(),
18144        }
18145    }
18146
18147    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18148        let mut r = review_round(true, 0, 1, 1, true, true);
18149        r.round = round;
18150        r.fix = notes.map(|n| FixRecord {
18151            agent: "fixer".to_owned(),
18152            addressed: Vec::new(),
18153            rejected: Vec::new(),
18154            notes: n.to_owned(),
18155            committed: true,
18156            failed: None,
18157            duration_ms: 0,
18158            continuation: None,
18159        });
18160        r
18161    }
18162
18163    #[test]
18164    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18165        let state = review_state(&["ignored"]);
18166        let f = facts(
18167            &[
18168                (
18169                    "fix(login): resolve PATH on macOS",
18170                    "Login shells skip rc files.",
18171                ),
18172                ("fix: address review", ""),
18173            ],
18174            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18175        );
18176        let m = pr_message_with(&state, 'A', Some(&f));
18177        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18178        assert!(
18179            m.body
18180                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
18181            "{}",
18182            m.body
18183        );
18184        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18185        assert!(m.body.contains("## Diff stat"), "{}", m.body);
18186        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18187        for banned in [
18188            "Review the work already",
18189            "no task statement",
18190            "Original task",
18191        ] {
18192            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18193        }
18194        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18195    }
18196
18197    #[test]
18198    fn pr_message_review_truncates_a_huge_first_commit_body() {
18199        let state = review_state(&["ignored"]);
18200        let f = facts(
18201            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18202            "s",
18203        );
18204        let m = pr_message_with(&state, 'A', Some(&f));
18205        assert!(m.body.len() < 30_000, "{}", m.body.len());
18206        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18207        assert!(
18208            m.body.contains("- ... 1 more commit(s)"),
18209            "{}",
18210            m.body.len()
18211        );
18212        assert!(m.body.ends_with("magi:candidate-a\n"));
18213    }
18214
18215    #[test]
18216    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18217        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18218        assert_eq!(m.title, "feat: x");
18219        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18220        assert!(!m.body.contains("Diff stat"), "{}", m.body);
18221        assert!(!m.body.contains("no task statement"), "{}", m.body);
18222    }
18223
18224    #[test]
18225    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18226        let mut state = review_state(&["日本語の件名"]);
18227        state.candidates[0].branch = "fix/macos-login-path".to_owned();
18228        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18229        state.candidates[0].branch = "機能/ログイン".to_owned();
18230        assert!(
18231            pr_message(&state, 'A')
18232                .title
18233                .starts_with("chore: land candidate A")
18234        );
18235    }
18236
18237    #[test]
18238    fn pr_message_review_fixes_survive_a_clean_final_round() {
18239        let mut state = review_state(&["feat: x"]);
18240        state.reviews = vec![
18241            round_with_notes(1, Some("handled the PATH case")),
18242            round_with_notes(2, None),
18243        ];
18244        let body = pr_message(&state, 'A').body;
18245        assert!(
18246            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18247            "{body}"
18248        );
18249        assert!(body.contains("### Round 1"), "{body}");
18250
18251        state.reviews = vec![
18252            round_with_notes(1, Some("first fix")),
18253            round_with_notes(2, Some("")),
18254            round_with_notes(3, Some("second fix")),
18255            round_with_notes(4, None),
18256        ];
18257        let body = pr_message(&state, 'A').body;
18258        assert!(
18259            body.contains("### Round 1") && !body.contains("first fix"),
18260            "{body}"
18261        );
18262        assert!(
18263            body.contains("### Round 3") && !body.contains("second fix"),
18264            "{body}"
18265        );
18266        assert!(body.contains("### Round 2"), "{body}");
18267    }
18268
18269    const FINAL: &str = "abcdef1234567890";
18270
18271    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18272        let mut r = round_with_notes(round, None);
18273        r.verified_head = verified.map(str::to_owned);
18274        r
18275    }
18276
18277    fn ok_gate(state: &mut RunState, head: Option<&str>) {
18278        state.gate = vec![CommandOutcome {
18279            command: "gate".to_owned(),
18280            code: Some(0),
18281            output_tail: String::new(),
18282            duration_ms: 0,
18283            resource_blocked: false,
18284        }];
18285        state.gate_ran = true;
18286        state.gate_head = head.map(str::to_owned);
18287    }
18288
18289    fn body_at(state: &RunState) -> String {
18290        pr_message_raw(
18291            state,
18292            'A',
18293            None,
18294            crate::pr_outcome::FinalHead::At(FINAL),
18295            ChangeNote::NotAsked,
18296        )
18297        .body
18298    }
18299
18300    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18301
18302    #[test]
18303    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18304        let mut state = review_state(&["feat: x"]);
18305        state.reviews = vec![
18306            round_with_notes(1, Some(SANDBOXED)),
18307            passing_round(2, Some(FINAL)),
18308        ];
18309        ok_gate(&mut state, Some(FINAL));
18310        let body = body_at(&state);
18311        assert!(!body.contains("Handled the PATH case."), "{body}");
18312        assert!(!body.contains("could not commit"), "{body}");
18313        assert!(!body.contains("could not run"), "{body}");
18314        assert!(
18315            body.contains("The fix is committed on the branch"),
18316            "{body}"
18317        );
18318        assert!(!body.contains("sandbox"), "{body}");
18319        assert!(
18320            body.contains("Tests (`verify.e2e`): passed on the final commit"),
18321            "{body}"
18322        );
18323        assert!(
18324            body.contains("Gate (`verify.gate`): passed on the final commit"),
18325            "{body}"
18326        );
18327        // The raw report is untouched in the record.
18328        assert!(
18329            state.reviews[0]
18330                .fix
18331                .as_ref()
18332                .unwrap()
18333                .notes
18334                .contains("could not commit")
18335        );
18336    }
18337
18338    #[test]
18339    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18340        let mut state = review_state(&["feat: x"]);
18341        let mut r1 = round_with_notes(1, Some(SANDBOXED));
18342        r1.fix.as_mut().unwrap().committed = false;
18343        let mut r2 = passing_round(2, Some(FINAL));
18344        r2.e2e[0].code = Some(1);
18345        state.reviews = vec![r1, r2];
18346        let body = body_at(&state);
18347        assert!(!body.contains("could not commit"), "{body}");
18348        assert!(!body.contains("could not run"), "{body}");
18349        assert!(body.contains("This round produced no commit"), "{body}");
18350        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18351        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18352        assert!(!body.contains("passed"), "{body}");
18353    }
18354
18355    #[test]
18356    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18357        let mut state = review_state(&["feat: x"]);
18358        state.reviews = vec![
18359            round_with_notes(1, Some(SANDBOXED)),
18360            passing_round(2, Some("0123456789abcdef")),
18361        ];
18362        ok_gate(&mut state, Some("0123456789abcdef"));
18363        let body = body_at(&state);
18364        assert!(!body.contains("HTTP tests could not run"), "{body}");
18365        // The commit claim is still settled by the structured record.
18366        assert!(!body.contains("could not commit"), "{body}");
18367        assert!(
18368            body.contains("passed on an earlier commit (`0123456`)"),
18369            "{body}"
18370        );
18371        assert!(!body.contains("passed on the final commit"), "{body}");
18372
18373        // A gate recorded before `gate_head` existed is "not recorded".
18374        ok_gate(&mut state, None);
18375        let body = body_at(&state);
18376        assert!(
18377            body.contains("passed on a commit that was not recorded"),
18378            "{body}"
18379        );
18380    }
18381
18382    #[test]
18383    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18384        let state = state_with_summary(
18385            "add retries to the client",
18386            "TITLE: feat: retries\n\nDid it.",
18387        );
18388        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18389        assert_eq!(m.title, "feat: retries");
18390        assert!(
18391            m.body.contains("<summary>Original task</summary>"),
18392            "{}",
18393            m.body
18394        );
18395        assert!(!m.body.contains("Commits under review"), "{}", m.body);
18396        assert!(
18397            m.body
18398                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18399            "{}",
18400            m.body
18401        );
18402    }
18403
18404    #[test]
18405    fn pr_message_review_bounds_a_long_english_subject() {
18406        let long = format!("feat: {}", "word ".repeat(100));
18407        let m = pr_message(&review_state(&[&long]), 'A');
18408        assert!(m.title.starts_with("feat: word"), "{}", m.title);
18409        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18410    }
18411
18412    #[test]
18413    fn pr_message_implementation_run_is_unchanged_by_review_support() {
18414        let state = state_with_summary("add retries\n\ndetails", "- did some things");
18415        let m = pr_message(&state, 'A');
18416        assert_eq!(m.title, "add retries");
18417        assert!(m.body.contains("<summary>Original task</summary>"));
18418        assert!(!m.body.contains("Commits under review"));
18419        assert_eq!(landing_subject_source(&state), state.instruction);
18420    }
18421
18422    #[test]
18423    fn review_run_squash_subject_is_the_change_not_the_prompt() {
18424        let state = review_state(&["feat: the change", "fix: typo"]);
18425        let source = landing_subject_source(&state);
18426        assert_eq!(land::merge_subject("", &source), "feat: the change");
18427        assert_eq!(
18428            land::merge_subject("magi: candidate A (uncommitted work)", &source),
18429            "feat: the change"
18430        );
18431        // An operator's rename still wins.
18432        assert_eq!(
18433            land::merge_subject("feat: renamed by hand", &source),
18434            "feat: renamed by hand"
18435        );
18436        let blank = review_state(&["日本語"]);
18437        assert!(
18438            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18439        );
18440    }
18441
18442    #[test]
18443    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18444        let state = review_state(&["feat: the change"]);
18445        let source = landing_subject_source(&state);
18446        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18447        assert_eq!(
18448            land::merge_subject(landing_title(&state, old), &source),
18449            "feat: the change"
18450        );
18451        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18452        let task = state_with_summary("add retries", "");
18453        assert_eq!(landing_title(&task, old), old);
18454    }
18455
18456    #[test]
18457    fn pr_message_describes_the_change_not_the_task() {
18458        let state = state_with_summary(
18459            "今回やってほしいこと: results projector を直す",
18460            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18461        );
18462        let m = pr_message(&state, 'A');
18463        assert_eq!(m.title, "fix(web): batch the runs list reads");
18464        assert!(
18465            m.body.starts_with("## Summary\n\n- reads run.json once"),
18466            "{}",
18467            m.body
18468        );
18469        assert!(!m.body.contains("TITLE:"), "{}", m.body);
18470        let task_at = m.body.find("今回やってほしいこと").unwrap();
18471        let details_at = m.body.find("<details>").unwrap();
18472        assert!(
18473            details_at < task_at,
18474            "the task lives inside <details>: {}",
18475            m.body
18476        );
18477        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18478        assert!(m.body.contains("magi:candidate-a"));
18479    }
18480
18481    #[test]
18482    fn pr_message_falls_back_to_the_task_without_a_title_line() {
18483        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18484        let m = pr_message(&state, 'A');
18485        assert_eq!(m.title, "add retries");
18486        assert!(
18487            m.body.contains("## Summary\n\n- did some things"),
18488            "{}",
18489            m.body
18490        );
18491
18492        let none = RunState::new(
18493            PathBuf::from("/repo"),
18494            "main".to_owned(),
18495            "abc1234".to_owned(),
18496            "add retries".to_owned(),
18497            Config::default(),
18498        );
18499        let m = pr_message(&none, 'A');
18500        assert_eq!(m.title, "add retries");
18501        assert!(!m.body.contains("## Summary"), "{}", m.body);
18502    }
18503
18504    #[test]
18505    fn pr_message_refuses_the_candidate_commit_subject() {
18506        for bad in [
18507            "TITLE: magi: candidate A (uncommitted work)",
18508            "TITLE: chore: stuff (uncommitted work)",
18509            "TITLE:   ",
18510        ] {
18511            let state = state_with_summary("add retries", bad);
18512            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18513        }
18514    }
18515
18516    #[test]
18517    fn pr_message_bounds_a_very_long_task_and_title() {
18518        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18519        let state = state_with_summary(&long, "- nothing");
18520        let m = pr_message(&state, 'A');
18521        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18522        assert!(!m.title.contains('\n'));
18523
18524        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18525        let m = pr_message(&state, 'A');
18526        assert!(m.title.starts_with("feat: "));
18527        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18528        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18529    }
18530
18531    fn long_title_of(instruction: &str) -> String {
18532        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18533    }
18534
18535    #[test]
18536    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18537        let first = "Make the landing path keep a readable title for long tasks";
18538        let line = format!(
18539            "{first}. {}",
18540            "And then keep going with more words ".repeat(20)
18541        );
18542        let t = long_title_of(&line);
18543        assert_eq!(t, first);
18544        assert!(!t.starts_with("chore: land"));
18545    }
18546
18547    #[test]
18548    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18549        let line = "word ".repeat(200);
18550        let t = long_title_of(&line);
18551        assert!(t.ends_with("word..."), "{t}");
18552        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18553    }
18554
18555    #[test]
18556    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18557        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18558            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18559        }
18560    }
18561
18562    #[test]
18563    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18564        let mut state = state_with_summary("再試行を追加する", "- nothing");
18565        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18566        state.config.graph.github_language = "ja".into();
18567        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18568    }
18569
18570    #[test]
18571    fn pr_message_title_limit_is_exact() {
18572        let at = "a".repeat(PR_TITLE_MAX);
18573        assert_eq!(long_title_of(&at), at);
18574        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18575        assert!(over.ends_with("..."), "{over}");
18576        assert_eq!(over.chars().count(), PR_TITLE_MAX);
18577    }
18578
18579    #[test]
18580    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18581        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18582        let t = long_title_of(&line);
18583        assert!(t.ends_with("..."), "{t}");
18584        assert!(t.is_ascii(), "{t}");
18585    }
18586
18587    #[test]
18588    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18589        let line = format!(
18590            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18591            "plus more filler words ".repeat(20)
18592        );
18593        let t = long_title_of(&line);
18594        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18595    }
18596
18597    #[test]
18598    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18599        let line = format!("\n\n# {}", "title words ".repeat(40));
18600        let state = state_with_summary(&line, "- nothing");
18601        let m = pr_message(&state, 'A');
18602        assert!(m.title.starts_with("title words"), "{}", m.title);
18603        assert_eq!(
18604            land::merge_subject(&m.title, &landing_subject_source(&state)),
18605            m.title
18606        );
18607        // An operator's rename wins untouched.
18608        assert_eq!(
18609            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18610            "feat: renamed by hand"
18611        );
18612    }
18613
18614    #[tokio::test]
18615    async fn github_text_rewrite_is_bounded_and_falls_back() {
18616        crate::run::pin_test_home();
18617        for (reply, accepted) in [
18618            (
18619                "TITLE: fix: retries\nAdd retries for failed requests.",
18620                true,
18621            ),
18622            (
18623                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18624                false,
18625            ),
18626            (
18627                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18628                true,
18629            ),
18630            (
18631                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18632                true,
18633            ),
18634            (
18635                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18636                true,
18637            ),
18638            (
18639                "TITLE: fix: retries\n日本語の説明です。 token=secret",
18640                false,
18641            ),
18642            ("TITLE: fix: retries\n", false),
18643        ] {
18644            let dir = tempfile::tempdir().unwrap();
18645            let mut runner = runner_at(RunStatus::Gating);
18646            runner.state = state_with_summary(
18647                "add retries",
18648                "TITLE: fix: retries\n日本語の説明を書きます。",
18649            );
18650            runner.state.repo = dir.path().to_owned();
18651            runner.state.candidates[0].worktree = dir.path().to_owned();
18652            let mut author = spec("alpha");
18653            author.command = vec![
18654                "sh".into(),
18655                "-c".into(),
18656                "printf '%s' \"$REWRITE_REPLY\"".into(),
18657            ];
18658            author.env.insert("REWRITE_REPLY".into(), reply.into());
18659            runner.state.config.agents = vec![author];
18660            seed_public_summary(&mut runner);
18661            let winner = runner.state.candidates[0].clone();
18662            let first = runner
18663                .guarded_pr_message(&winner, None, true)
18664                .await
18665                .unwrap();
18666            let message = if accepted {
18667                first.expect("a passing rewrite never asks")
18668            } else {
18669                // Withheld: the run parks on a question and posts nothing yet.
18670                assert!(first.is_none());
18671                assert!(runner.state.parked);
18672                let store = ask::Questions::open();
18673                let asked: Vec<_> = store
18674                    .list()
18675                    .into_iter()
18676                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18677                    .collect();
18678                assert_eq!(asked.len(), 1);
18679                assert!(!asked[0].detail.contains("secret"));
18680                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18681                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18682                assert_eq!(
18683                    asked[0]
18684                        .choices
18685                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18686                    rewrite.is_some()
18687                );
18688                // Silence: the question lapses and the neutral text is used,
18689                // without a second rewrite call or a second question.
18690                store
18691                    .update(&asked[0].id, |q| {
18692                        q.abandon("test");
18693                        Ok(())
18694                    })
18695                    .unwrap();
18696                let again = runner
18697                    .guarded_pr_message(&winner, None, true)
18698                    .await
18699                    .unwrap()
18700                    .expect("an abandoned question falls back");
18701                let n = store
18702                    .list()
18703                    .into_iter()
18704                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18705                    .count();
18706                assert_eq!(n, 1, "asked once per rejected text");
18707                again
18708            };
18709            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18710            assert!(!message.body.contains("secret"));
18711            assert!(!message.body.contains("prefix"));
18712            assert!(!message.title.contains("secret"));
18713            assert_eq!(
18714                message.body.contains("Add retries for failed requests."),
18715                accepted
18716            );
18717            assert_eq!(
18718                runner.state.seats["github-text-rewrite"].turns, 1,
18719                "only one rewrite invocation"
18720            );
18721            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18722            if accepted {
18723                assert!(runner.state.github_text.is_none());
18724                assert!(!runner.state.parked);
18725                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18726                assert!(message.body.contains("## Final verification"));
18727                assert!(message.body.contains("Evidence: final diff"));
18728                assert!(
18729                    message
18730                        .body
18731                        .contains(&format!("magi:run/{}", runner.state.id))
18732                );
18733            }
18734            if !accepted {
18735                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18736                assert!(
18737                    message
18738                        .body
18739                        .contains(&format!("magi:run/{}", runner.state.id))
18740                );
18741            }
18742        }
18743    }
18744
18745    fn scripted(id: &str, reply: &str) -> AgentSpec {
18746        let mut a = spec(id);
18747        a.command = vec![
18748            "sh".into(),
18749            "-c".into(),
18750            "printf '%s' \"$REWRITE_REPLY\"".into(),
18751        ];
18752        a.env.insert("REWRITE_REPLY".into(), reply.into());
18753        a
18754    }
18755
18756    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18757    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18758
18759    /// Gate tests inject generated public output, not internal reports.
18760    fn seed_public_summary(runner: &mut Runner) {
18761        let repo = &runner.state.repo;
18762        let g = |args: &[&str]| {
18763            let out = std::process::Command::new("git")
18764                .args(args)
18765                .current_dir(repo)
18766                .quiet()
18767                .output()
18768                .unwrap();
18769            assert!(out.status.success(), "git {args:?}");
18770            String::from_utf8_lossy(&out.stdout).trim().to_owned()
18771        };
18772        g(&["init", "-q", "-b", "main"]);
18773        g(&[
18774            "-c",
18775            "user.name=Test",
18776            "-c",
18777            "user.email=test@example.invalid",
18778            "commit",
18779            "-q",
18780            "--allow-empty",
18781            "-m",
18782            "feat: fixture",
18783        ]);
18784        let head = g(&["rev-parse", "HEAD"]);
18785        g(&["update-ref", "refs/remotes/origin/main", &head]);
18786        let evidence = crate::pr_outcome::summary_evidence(
18787            &runner.state,
18788            crate::pr_outcome::FinalHead::At(&head),
18789        );
18790        let generated = &runner.state.candidates[0].summary;
18791        runner.state.change_summary = Some(crate::run::ChangeSummary {
18792            base: head.clone(),
18793            head,
18794            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18795            title: summary_title(generated).unwrap(),
18796            body: summary_without_title(generated),
18797        });
18798    }
18799
18800    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18801        let mut runner = runner_at(RunStatus::Gating);
18802        runner.state = state_with_summary("add retries", JA_SUMMARY);
18803        if review_only {
18804            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18805        }
18806        runner.state.repo = dir.to_owned();
18807        runner.state.candidates[0].worktree = dir.to_owned();
18808        runner.state.config.agents = agents;
18809        seed_public_summary(&mut runner);
18810        runner
18811    }
18812
18813    fn gate_questions(runner: &Runner) -> usize {
18814        ask::Questions::open()
18815            .list()
18816            .into_iter()
18817            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18818            .count()
18819    }
18820
18821    #[tokio::test]
18822    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18823        crate::run::pin_test_home();
18824        let dir = tempfile::tempdir().unwrap();
18825        let mut runner = rewrite_runner(
18826            dir.path(),
18827            true,
18828            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18829        );
18830        runner.state.config.roles.language_judge =
18831            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18832        let winner = runner.state.candidates[0].clone();
18833        let msg = runner
18834            .guarded_pr_message(&winner, None, true)
18835            .await
18836            .unwrap()
18837            .expect("an English rewrite posts without asking");
18838        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18839        assert!(msg.body.contains("<details>kept</details>"));
18840        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18841        assert_eq!(gate_questions(&runner), 0);
18842        assert!(runner.state.github_text.is_none());
18843        assert!(!runner.state.seats.contains_key("impl-A"));
18844        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18845    }
18846
18847    #[tokio::test]
18848    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18849        crate::run::pin_test_home();
18850        let dir = tempfile::tempdir().unwrap();
18851        let mut runner = rewrite_runner(
18852            dir.path(),
18853            false,
18854            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18855        );
18856        runner.state.config.roles.language_judge =
18857            Some(crate::config::AgentChoice::One("beta".into()));
18858        runner.state.seats.insert(
18859            "impl-A".into(),
18860            SeatState::new("impl-A", "alpha", runner.state.seed),
18861        );
18862        let winner = runner.state.candidates[0].clone();
18863        let msg = runner
18864            .guarded_pr_message(&winner, None, true)
18865            .await
18866            .unwrap()
18867            .expect("rewritten");
18868        assert!(msg.body.contains("<details>kept</details>"));
18869        assert_eq!(runner.state.seats["impl-A"].turns, 0);
18870        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18871        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18872    }
18873
18874    #[tokio::test]
18875    async fn rewrite_reuses_a_resumable_implementer_session() {
18876        crate::run::pin_test_home();
18877        let dir = tempfile::tempdir().unwrap();
18878        let mut runner = rewrite_runner(
18879            dir.path(),
18880            false,
18881            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18882        );
18883        runner.state.config.roles.language_judge =
18884            Some(crate::config::AgentChoice::One("beta".into()));
18885        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18886        seat.turns = 1;
18887        runner.state.seats.insert("impl-A".into(), seat);
18888        let winner = runner.state.candidates[0].clone();
18889        runner
18890            .guarded_pr_message(&winner, None, true)
18891            .await
18892            .unwrap()
18893            .expect("rewritten");
18894        assert_eq!(runner.state.seats["impl-A"].turns, 2);
18895        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18896    }
18897
18898    #[tokio::test]
18899    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
18900        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
18901        // session, was out of quota, and nothing else was asked.
18902        crate::run::pin_test_home();
18903        let dir = tempfile::tempdir().unwrap();
18904        let mut exhausted = spec("alpha");
18905        exhausted.command = vec!["false".into()];
18906        let mut runner = rewrite_runner(
18907            dir.path(),
18908            false,
18909            vec![exhausted, scripted("beta", EN_REWRITE)],
18910        );
18911        // The implementer's own agent is in the role too: it is not asked twice.
18912        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18913            "alpha".into(),
18914            "beta".into(),
18915        ]));
18916        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18917        seat.turns = 1;
18918        runner.state.seats.insert("impl-A".into(), seat);
18919        let winner = runner.state.candidates[0].clone();
18920        let msg = runner
18921            .guarded_pr_message(&winner, None, true)
18922            .await
18923            .unwrap()
18924            .expect("the role rewrites after the implementer fails");
18925        assert!(msg.body.contains("`src/net.rs`"));
18926        assert_eq!(
18927            runner.state.seats["impl-A"].turns, 2,
18928            "the implementer was tried once"
18929        );
18930        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18931        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18932    }
18933
18934    #[tokio::test]
18935    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18936        crate::run::pin_test_home();
18937        let dir = tempfile::tempdir().unwrap();
18938        let mut broken = spec("bad");
18939        broken.command = vec!["false".into()];
18940        let mut runner = rewrite_runner(
18941            dir.path(),
18942            true,
18943            vec![
18944                broken,
18945                scripted("noformat", "Add retries."),
18946                scripted("good", EN_REWRITE),
18947            ],
18948        );
18949        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18950            "bad".into(),
18951            "noformat".into(),
18952            "good".into(),
18953        ]));
18954        let winner = runner.state.candidates[0].clone();
18955        let msg = runner
18956            .guarded_pr_message(&winner, None, true)
18957            .await
18958            .unwrap()
18959            .expect("the third agent rewrites");
18960        assert!(msg.body.contains("`src/net.rs`"));
18961        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18962        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18963    }
18964
18965    #[tokio::test]
18966    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18967        crate::run::pin_test_home();
18968        // (agents, role): still Japanese; no TITLE; nothing resolvable.
18969        for (agents, role, saved) in [
18970            (
18971                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18972                Some("beta"),
18973                true,
18974            ),
18975            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18976            (Vec::new(), None, false),
18977        ] {
18978            let dir = tempfile::tempdir().unwrap();
18979            let mut runner = rewrite_runner(dir.path(), true, agents);
18980            runner.state.config.roles.language_judge =
18981                role.map(|r| crate::config::AgentChoice::One(r.into()));
18982            let winner = runner.state.candidates[0].clone();
18983            let first = runner
18984                .guarded_pr_message(&winner, None, true)
18985                .await
18986                .unwrap();
18987            assert!(first.is_none(), "withheld text parks the run");
18988            assert!(runner.state.parked);
18989            assert_eq!(gate_questions(&runner), 1);
18990            let g = runner.state.github_text.as_ref().unwrap();
18991            assert_eq!(g.rewrite.is_some(), saved);
18992            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18993        }
18994    }
18995
18996    #[tokio::test]
18997    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18998        crate::run::pin_test_home();
18999        let dir = tempfile::tempdir().unwrap();
19000        let mut runner = runner_at(RunStatus::Gating);
19001        runner.state =
19002            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
19003        runner.state.repo = dir.path().to_owned();
19004        runner.state.candidates[0].worktree = dir.path().to_owned();
19005        let mut author = spec("alpha");
19006        author.command = vec![
19007            "sh".into(),
19008            "-c".into(),
19009            "printf '%s' \"$REWRITE_REPLY\"".into(),
19010        ];
19011        author.env.insert(
19012            "REWRITE_REPLY".into(),
19013            format!(
19014                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19015                "English sentence. ".repeat(600)
19016            ),
19017        );
19018        runner.state.config.agents = vec![author];
19019        seed_public_summary(&mut runner);
19020        let winner = runner.state.candidates[0].clone();
19021        assert!(
19022            runner
19023                .guarded_pr_message(&winner, None, true)
19024                .await
19025                .unwrap()
19026                .is_none()
19027        );
19028        let record = runner.state.github_text.clone().unwrap();
19029        let rewrite = record.rewrite.clone().unwrap();
19030        assert_eq!(rewrite.failed, ["body-language"]);
19031        assert!(
19032            rewrite
19033                .body
19034                .ends_with(&format!("magi:run/{}", runner.state.id))
19035        );
19036        let store = ask::Questions::open();
19037        let q = store.get(record.question.as_ref().unwrap()).unwrap();
19038        assert!(
19039            q.choices
19040                .contains(&crate::github_text::USE_REWRITE.to_owned())
19041        );
19042        for label in [
19043            "ORIGINAL",
19044            "FALLBACK",
19045            "REWRITE",
19046            "Rules still failed: body-language",
19047            "truncated",
19048        ] {
19049            assert!(q.detail.contains(label), "{label}");
19050        }
19051        let artifact = std::fs::read_to_string(crate::run::artifact_path(
19052            &runner.state,
19053            &crate::github_text::artifact_name(&record.fingerprint),
19054        ))
19055        .unwrap();
19056        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19057        // Restart from serialized state, then retry after a bad owner title.
19058        runner.state =
19059            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19060        store
19061            .update(&q.id, |q| {
19062                q.say("まだ日本語")?;
19063                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19064            })
19065            .unwrap();
19066        assert!(
19067            runner
19068                .guarded_pr_message(&winner, None, true)
19069                .await
19070                .unwrap()
19071                .is_none()
19072        );
19073        let retry = runner.state.github_text.as_ref().unwrap();
19074        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19075        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19076        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19077        assert!(
19078            q.posting_comparison
19079                .as_ref()
19080                .unwrap()
19081                .matches_detail(&q.detail)
19082        );
19083        store
19084            .update(&q.id, |q| {
19085                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19086            })
19087            .unwrap();
19088        let message = runner
19089            .guarded_pr_message(&winner, None, true)
19090            .await
19091            .unwrap()
19092            .unwrap();
19093        assert_eq!(message.title, rewrite.title);
19094        assert_eq!(message.body, rewrite.body);
19095        assert_eq!(
19096            runner.state.github_text.as_ref().unwrap().choice,
19097            Some(GithubTextChoice::Rewrite)
19098        );
19099        let again = runner
19100            .guarded_pr_message(&winner, None, true)
19101            .await
19102            .unwrap()
19103            .unwrap();
19104        assert_eq!(again.title, message.title);
19105        assert_eq!(again.body, message.body);
19106        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19107        assert!(
19108            !runner
19109                .state
19110                .events
19111                .iter()
19112                .any(|e| e.message.contains(&rewrite.title)
19113                    || e.message.contains("English sentence."))
19114        );
19115    }
19116
19117    #[tokio::test]
19118    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19119        crate::run::pin_test_home();
19120        let dir = tempfile::tempdir().unwrap();
19121        let mut runner = runner_at(RunStatus::Gating);
19122        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19123        runner.state.repo = dir.path().to_owned();
19124        runner.state.candidates[0].worktree = dir.path().to_owned();
19125        seed_public_summary(&mut runner);
19126        let winner = runner.state.candidates[0].clone();
19127        assert!(
19128            runner
19129                .guarded_pr_message(&winner, None, true)
19130                .await
19131                .unwrap()
19132                .is_none()
19133        );
19134        let record = runner.state.github_text.as_ref().unwrap();
19135        assert!(record.rewrite.is_none());
19136        // Records written before the additive field deserialize with no rewrite.
19137        let mut old = serde_json::to_value(record).unwrap();
19138        old.as_object_mut().unwrap().remove("rewrite");
19139        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19140        assert!(old.rewrite.is_none());
19141        let id = record.question.clone().unwrap();
19142        ask::Questions::open()
19143            .update(&id, |q| {
19144                q.choices.push(crate::github_text::USE_REWRITE.into());
19145                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19146            })
19147            .unwrap();
19148        let message = runner
19149            .guarded_pr_message(&winner, None, true)
19150            .await
19151            .unwrap()
19152            .unwrap();
19153        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19154        assert_eq!(
19155            runner.state.github_text.as_ref().unwrap().choice,
19156            Some(GithubTextChoice::Fallback)
19157        );
19158    }
19159
19160    #[tokio::test]
19161    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19162        crate::run::pin_test_home();
19163        let dir = tempfile::tempdir().unwrap();
19164        let mut runner = runner_at(RunStatus::Gating);
19165        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19166        runner.state.repo = dir.path().to_owned();
19167        runner.state.candidates[0].worktree = dir.path().to_owned();
19168        seed_public_summary(&mut runner);
19169        let winner = runner.state.candidates[0].clone();
19170        assert!(
19171            runner
19172                .guarded_pr_message(&winner, None, true)
19173                .await
19174                .unwrap()
19175                .is_none()
19176        );
19177        let store = ask::Questions::open();
19178        let find = |run: &str| {
19179            store
19180                .list()
19181                .into_iter()
19182                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19183                .unwrap()
19184        };
19185        let run = runner.state.id.clone();
19186        let q = find(&run);
19187        let fp = runner
19188            .state
19189            .github_text
19190            .as_ref()
19191            .unwrap()
19192            .fingerprint
19193            .clone();
19194        let name = crate::github_text::artifact_name(&fp);
19195        let path = crate::run::artifact_path(&runner.state, &name);
19196        assert!(q.detail.contains(&path.display().to_string()));
19197        assert!(q.detail.contains("title-language: title"));
19198        let saved = std::fs::read_to_string(&path).expect("artifact written");
19199        assert!(saved.contains("日本語のタイトル"));
19200        assert!(q.cwd.is_none());
19201        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19202        // A bad replacement is refused and asked about once more.
19203        let say = |id: &str, text: &str, choice: &str| {
19204            store
19205                .update(id, |q| {
19206                    q.thread.push(ask::Turn {
19207                        who: ask::Who::Operator,
19208                        body: text.to_owned(),
19209                        at: jiff::Timestamp::now(),
19210                        note: None,
19211                    });
19212                    q.answer(ask::Answer::Choice(choice.to_owned()))
19213                })
19214                .unwrap();
19215        };
19216        say(&q.id, "まだ日本語", "use my text");
19217        assert!(
19218            runner
19219                .guarded_pr_message(&winner, None, true)
19220                .await
19221                .unwrap()
19222                .is_none()
19223        );
19224        let second = find(&run);
19225        assert_ne!(second.id, q.id);
19226        assert!(second.detail.contains("did not pass"));
19227        // The retry still compares the original text; the rejected
19228        // replacement is named by category only.
19229        assert!(second.detail.contains("日本語のタイトル"));
19230        assert!(!second.detail.contains("まだ日本語"));
19231        assert!(second.detail.contains(&path.display().to_string()));
19232        say(&second.id, "fix: retry failed requests", "use my text");
19233        let message = runner
19234            .guarded_pr_message(&winner, None, true)
19235            .await
19236            .unwrap()
19237            .expect("a vetted replacement is posted");
19238        assert_eq!(message.title, "fix: retry failed requests");
19239        assert_eq!(
19240            runner
19241                .state
19242                .github_text
19243                .as_ref()
19244                .unwrap()
19245                .chosen_title
19246                .as_deref(),
19247            Some("fix: retry failed requests")
19248        );
19249        assert!(runner.state.github_text.as_ref().unwrap().resolved);
19250        // Resumed again: the saved decision is applied, nothing is re-asked.
19251        let again = runner
19252            .guarded_pr_message(&winner, None, true)
19253            .await
19254            .unwrap()
19255            .unwrap();
19256        assert_eq!(again.title, message.title);
19257        assert_eq!(
19258            store
19259                .list()
19260                .into_iter()
19261                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19262                .count(),
19263            2
19264        );
19265    }
19266
19267    #[test]
19268    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19269        // What magi itself writes stays English under any configured language,
19270        // so a future localisation of these headings fails here. (The agents'
19271        // own text is also checked by the posting gate.)
19272        let mut state = state_with_summary(
19273            "add retries",
19274            "TITLE: fix(web): batch reads\n- reads run.json once",
19275        );
19276        state.config.graph.language = "ja".to_owned();
19277        let m = pr_message(&state, 'A');
19278        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19279        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19280
19281        // The task is the operator's own text: it goes in untouched, and the
19282        // fallback title (no summary) may be in its language too.
19283        let task = "今回やってほしいこと: results projector を直す";
19284        let mut state = state_with_summary(task, "- no title line");
19285        state.config.graph.language = "ja".to_owned();
19286        let m = pr_message(&state, 'A');
19287        assert_eq!(
19288            m.title,
19289            format!("chore: land candidate A of run {}", state.id)
19290        );
19291        assert!(
19292            m.body.contains(&format!(
19293                "<summary>Original task</summary>\n\n{task}\n\n</details>"
19294            )),
19295            "{}",
19296            m.body
19297        );
19298    }
19299
19300    #[test]
19301    fn pr_message_scrubs_home_paths_and_addresses() {
19302        let state = state_with_summary(
19303            "fix it in /Users/someone/src/x",
19304            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19305        );
19306        let m = pr_message(&state, 'A');
19307        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19308            assert!(!m.body.contains(leak), "{}", m.body);
19309        }
19310        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19311    }
19312
19313    #[test]
19314    fn pr_message_survives_a_task_that_closes_details() {
19315        let state = state_with_summary("a </details> b", "TITLE: fix: x");
19316        let m = pr_message(&state, 'A');
19317        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19318    }
19319
19320    #[test]
19321    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19322        let cmd = manual_merge_command(
19323            MergeStyle::Squash,
19324            Path::new("/repo"),
19325            "b",
19326            "fix: \"quoted\" $(x) `y`\n\nbody",
19327        );
19328        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19329    }
19330
19331    #[test]
19332    fn manual_merge_command_matches_the_configured_style() {
19333        let repo = Path::new("/repo");
19334        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19335
19336        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19337        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19338
19339        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19340        assert_eq!(
19341            squash,
19342            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19343             \"Merge magi run 0832 (candidate A)\""
19344        );
19345
19346        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19347        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19348    }
19349
19350    #[test]
19351    fn a_nudge_gets_a_quarter_of_the_budget() {
19352        // The judge and implement budgets magi ships with.
19353        assert_eq!(retry_budget(secs(1200), true), secs(300));
19354        assert_eq!(retry_budget(secs(3600), true), secs(900));
19355    }
19356
19357    #[test]
19358    fn a_resent_prompt_keeps_the_whole_budget() {
19359        // The seat kept no context, so the retry is the original job again and
19360        // shortening it would only guarantee a second failure.
19361        assert_eq!(retry_budget(secs(1200), false), secs(1200));
19362        assert_eq!(retry_budget(secs(60), false), secs(60));
19363    }
19364
19365    #[test]
19366    fn the_floor_never_exceeds_the_original_budget() {
19367        // A short configured timeout must not be *raised* by the floor: the
19368        // operator asked for a bound, and a retry may not outlast the attempt
19369        // it is retrying.
19370        assert_eq!(retry_budget(secs(60), true), secs(60));
19371        assert_eq!(retry_budget(secs(480), true), secs(120));
19372        assert_eq!(retry_budget(secs(0), true), secs(0));
19373    }
19374
19375    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19376        agent::CommandEvidence {
19377            id: "item1".to_owned(),
19378            description: "cargo test".to_owned(),
19379            exit_code,
19380            result_summary: String::new(),
19381            source: "codex".to_owned(),
19382        }
19383    }
19384
19385    #[test]
19386    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19387        // No evidence is not the same fact as unconfirmed evidence: a
19388        // backend with no adapter, or a reply that ran no commands at all,
19389        // must not be misread as carrying a dangling job.
19390        assert!(!has_unconfirmed_command(&[]));
19391    }
19392
19393    #[test]
19394    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19395        // Deliberately not a check on the exit code's *value*: a fixer
19396        // legitimately runs something that fails mid-iteration before it
19397        // succeeds, and that must never by itself reopen a valid report.
19398        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19399        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19400        assert!(!has_unconfirmed_command(&[
19401            evidence(Some(0)),
19402            evidence(Some(101))
19403        ]));
19404    }
19405
19406    #[test]
19407    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19408        assert!(has_unconfirmed_command(&[
19409            evidence(Some(0)),
19410            evidence(None)
19411        ]));
19412    }
19413
19414    #[test]
19415    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19416        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19417        assert_eq!(
19418            verified_noop_claim(true, &[], text).as_deref(),
19419            Some("already fixed by b32cfc4, on main.")
19420        );
19421    }
19422
19423    #[test]
19424    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19425        // A timeout or a bad exit code reads as the ordinary loss it is,
19426        // whatever the reply's own prose claims.
19427        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19428        assert!(verified_noop_claim(false, &[], text).is_none());
19429    }
19430
19431    #[test]
19432    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19433        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19434        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19435        // A confirmed command alongside the marker is fine.
19436        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19437    }
19438
19439    #[test]
19440    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19441        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19442    }
19443
19444    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
19445    /// pair, in order, labelled A, B, C, ...
19446    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19447        runner.state.candidates = shape
19448            .iter()
19449            .enumerate()
19450            .map(|(i, &(empty, verified))| Candidate {
19451                index: i,
19452                label: (b'A' + i as u8) as char,
19453                agent: "sonnet".to_owned(),
19454                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19455                worktree: PathBuf::from(format!("/wt/{i}")),
19456                summary: String::new(),
19457                stat: String::new(),
19458                files: 0,
19459                commits: 0,
19460                empty,
19461                failed: None,
19462                verified_noop: verified.map(str::to_owned),
19463                duration_ms: 0,
19464                folded: false,
19465            })
19466            .collect();
19467    }
19468
19469    #[test]
19470    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19471        ask_test_home();
19472        let mut runner = runner_at(RunStatus::Implementing);
19473        set_candidates(
19474            &mut runner,
19475            &[
19476                (true, Some("already on main at b32cfc4")),
19477                (true, Some("same fix, see the existing test")),
19478            ],
19479        );
19480
19481        runner
19482            .after_implement()
19483            .expect("a verified no-op is not an error");
19484
19485        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19486    }
19487
19488    #[test]
19489    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19490        ask_test_home();
19491        let mut runner = runner_at(RunStatus::Implementing);
19492        // Candidate A declares a verified no-op; candidate B simply wrote
19493        // nothing and said nothing about why. One candidate's claim is not
19494        // the whole run's agreement.
19495        set_candidates(
19496            &mut runner,
19497            &[(true, Some("already on main at b32cfc4")), (true, None)],
19498        );
19499
19500        let err = runner
19501            .after_implement()
19502            .expect_err("an unverified empty candidate must still fail the run");
19503
19504        assert!(
19505            err.to_string().contains("no candidate produced a change"),
19506            "{err}"
19507        );
19508        assert_eq!(runner.state.status, RunStatus::Failed);
19509    }
19510
19511    #[test]
19512    fn after_implement_still_fails_an_ordinary_all_empty_run() {
19513        ask_test_home();
19514        let mut runner = runner_at(RunStatus::Implementing);
19515        set_candidates(&mut runner, &[(true, None), (true, None)]);
19516
19517        let err = runner
19518            .after_implement()
19519            .expect_err("no candidate declared anything; this is an ordinary failure");
19520
19521        assert!(
19522            err.to_string().contains("no candidate produced a change"),
19523            "{err}"
19524        );
19525        assert_eq!(runner.state.status, RunStatus::Failed);
19526    }
19527}
19528
19529/// A graft rollback whose restore could not be confirmed (see
19530/// [`Runner::graft`]).
19531#[derive(Debug)]
19532struct GraftRestoreFailed(String);
19533
19534impl std::fmt::Display for GraftRestoreFailed {
19535    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19536        write!(
19537            f,
19538            "could not restore the winner to its pre-graft commit: {}",
19539            self.0
19540        )
19541    }
19542}
19543
19544impl std::error::Error for GraftRestoreFailed {}
19545
19546/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
19547/// not the agent: they never draw on an agent's own timeout, and running out of
19548/// them never reads as an agent failure.
19549#[cfg(test)]
19550mod resource_wait_tests {
19551    use super::*;
19552    use std::collections::BTreeMap;
19553
19554    fn sh(id: &str, script: &str) -> AgentSpec {
19555        AgentSpec {
19556            id: id.to_owned(),
19557            kind: crate::config::AgentKind::Command,
19558            model: None,
19559            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19560            extra_args: Vec::new(),
19561            env: BTreeMap::new(),
19562            prompt_delivery: None,
19563            max_turn_secs: None,
19564        }
19565    }
19566
19567    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19568        SeatJob {
19569            seat: SeatState::new("impl-A", &spec.id, 1),
19570            spec,
19571            cwd: dir.to_path_buf(),
19572            prompt: "do it".to_owned(),
19573            timeout,
19574            allow_write: true,
19575            sessions: false,
19576            artifacts: dir.join("art"),
19577            stem: "impl-A".to_owned(),
19578            handover: None,
19579        }
19580    }
19581
19582    fn state(resource_wait_secs: u64) -> RunState {
19583        let mut config = Config::default();
19584        config.graph.timeout_resource_wait = resource_wait_secs;
19585        RunState::new(
19586            PathBuf::from("/repo"),
19587            "main".to_owned(),
19588            "abc1234".to_owned(),
19589            "task".to_owned(),
19590            config,
19591        )
19592    }
19593
19594    fn hold(cache: &Path) -> crate::cache::Guard {
19595        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19596        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19597            crate::cache::AcquireOutcome::Acquired(g) => g,
19598            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19599        }
19600    }
19601
19602    async fn run(
19603        jobs: Vec<SeatJob>,
19604        sem: Arc<Semaphore>,
19605        cache: Option<&Path>,
19606        st: &mut RunState,
19607    ) -> Vec<(usize, SeatState, AgentOutcome)> {
19608        let prompts = Prompts::default();
19609        let ctx = WaveCtx {
19610            run: "r",
19611            node: "implement",
19612            prompts: &prompts,
19613            cache,
19614            round: None,
19615            carry_seats: false,
19616            pauses: None,
19617        };
19618        wave(jobs, sem, &ctx, st, 0).await
19619    }
19620
19621    #[tokio::test]
19622    async fn verification_cache_wait_preserves_the_command_budget() {
19623        crate::run::pin_test_home();
19624        let tmp = tempfile::tempdir().unwrap();
19625        let cache = tmp.path().join("target");
19626        let guard = hold(&cache);
19627        let release = tokio::spawn(async move {
19628            tokio::time::sleep(Duration::from_millis(100)).await;
19629            guard.release();
19630        });
19631        let mut st = state(10);
19632        let budget = Duration::from_millis(50);
19633        let wait = repository_wait(&st, None);
19634        let (outcomes, _) = with_cache_lease(
19635            &mut st,
19636            Some(&cache),
19637            "gate",
19638            "gate",
19639            tmp.path(),
19640            "abc1234",
19641            budget,
19642            "test gate",
19643            &wait,
19644            |_, available| async move {
19645                assert_eq!(
19646                    available, budget,
19647                    "contention must not shorten verification"
19648                );
19649                (Vec::new(), false, Vec::new())
19650            },
19651        )
19652        .await
19653        .unwrap();
19654        release.await.unwrap();
19655        assert!(outcomes.is_empty(), "{outcomes:?}");
19656        assert!(
19657            st.events
19658                .iter()
19659                .any(|event| event.message.contains("waiting for"))
19660        );
19661        assert!(st.resource_wait.is_none());
19662        // The verification body no longer owns the lease after returning.
19663        hold(&cache).release();
19664    }
19665
19666    #[tokio::test]
19667    async fn verification_cache_wait_expires_without_starting_commands() {
19668        crate::run::pin_test_home();
19669        let tmp = tempfile::tempdir().unwrap();
19670        let cache = tmp.path().join("target");
19671        let _held = hold(&cache);
19672        let mut st = state(0);
19673        let wait = repository_wait(&st, None);
19674        let (outcomes, _) = with_cache_lease(
19675            &mut st,
19676            Some(&cache),
19677            "gate",
19678            "gate",
19679            tmp.path(),
19680            "abc1234",
19681            Duration::from_secs(60),
19682            "test gate",
19683            &wait,
19684            |_, _| async { panic!("commands must not start without the cache lease") },
19685        )
19686        .await
19687        .unwrap();
19688        assert_eq!(outcomes.len(), 1);
19689        assert!(outcomes[0].resource_blocked);
19690        assert!(st.resource_wait.is_none());
19691        assert!(matches!(
19692            st.resource_block.unwrap().cause,
19693            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19694        ));
19695    }
19696
19697    #[tokio::test]
19698    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19699        crate::run::pin_test_home();
19700        let tmp = tempfile::tempdir().unwrap();
19701        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19702        let cache = tmp.path().join("cache");
19703        let mut state = state(21600);
19704        let wait = repository_wait(&state, None);
19705        let (outcomes, _) = with_cache_lease(
19706            &mut state,
19707            Some(&cache),
19708            "gate",
19709            "gate",
19710            tmp.path(),
19711            "head",
19712            Duration::from_secs(60),
19713            "gate",
19714            &wait,
19715            |_, _| async { panic!("freshness failure must not start verification") },
19716        )
19717        .await
19718        .unwrap();
19719        assert_eq!(outcomes.len(), 1);
19720        assert!(outcomes[0].resource_blocked);
19721        let block = state.resource_block.unwrap();
19722        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19723        assert!(!block.description().contains("timed out"));
19724    }
19725
19726    #[tokio::test]
19727    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19728        crate::run::pin_test_home();
19729        let tmp = tempfile::tempdir().unwrap();
19730        let cache = tmp.path().join("cache");
19731        let _held = hold(&cache);
19732        let mut state = state(21600);
19733        let pause = Pause::new();
19734        let interrupt = Pause::new();
19735        let cause = ParkCause::Operator;
19736        pause.park_for(cause.clone());
19737        let prompts = Prompts::default();
19738        let ctx = WaveCtx {
19739            run: "test",
19740            node: "fix",
19741            prompts: &prompts,
19742            cache: Some(&cache),
19743            round: None,
19744            carry_seats: false,
19745            pauses: Some((&pause, &interrupt)),
19746        };
19747        let result = wave(
19748            vec![job(
19749                sh("writer", "exit 99"),
19750                tmp.path(),
19751                Duration::from_secs(1),
19752            )],
19753            Arc::new(Semaphore::new(1)),
19754            &ctx,
19755            &mut state,
19756            0,
19757        )
19758        .await;
19759        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19760        assert_eq!(result[0].1.turns, 0);
19761        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19762        assert_eq!(
19763            state.resource_block.unwrap().cause,
19764            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19765        );
19766        assert!(state.active.is_empty());
19767    }
19768
19769    #[tokio::test]
19770    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19771        crate::run::pin_test_home();
19772        let tmp = tempfile::tempdir().unwrap();
19773        let cache = tmp.path().join("target-a");
19774        let guard = hold(&cache);
19775        // Held for longer than the agent's whole 1s budget.
19776        tokio::spawn(async move {
19777            tokio::time::sleep(Duration::from_millis(1500)).await;
19778            guard.release();
19779        });
19780        let mut st = state(60);
19781        let results = run(
19782            vec![job(
19783                sh("alpha", "echo ok"),
19784                tmp.path(),
19785                Duration::from_secs(1),
19786            )],
19787            Arc::new(Semaphore::new(1)),
19788            Some(&cache),
19789            &mut st,
19790        )
19791        .await;
19792        assert!(
19793            matches!(results[0].2, AgentOutcome::Ok(_)),
19794            "the wait must not be charged to the agent"
19795        );
19796        assert!(st.resource_block.is_none());
19797        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19798        assert!(st.resource_wait.is_none() && st.active.is_empty());
19799    }
19800
19801    #[tokio::test]
19802    #[cfg(unix)]
19803    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19804        use std::os::fd::AsRawFd as _;
19805        crate::run::pin_test_home();
19806        let tmp = tempfile::tempdir().unwrap();
19807        crate::git::git(tmp.path(), &["init"]).await.unwrap();
19808        let common = tmp.path().join(".git");
19809        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19810        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19811        assert_eq!(
19812            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19813            0
19814        );
19815        let mut st = state(1);
19816        st.config.coordination = crate::config::test_coordination();
19817        let mut reader = job(
19818            sh("reader", "echo must-not-run"),
19819            tmp.path(),
19820            Duration::from_secs(30),
19821        );
19822        reader.allow_write = false;
19823        let results = run(
19824            vec![reader.clone()],
19825            Arc::new(Semaphore::new(1)),
19826            None,
19827            &mut st,
19828        )
19829        .await;
19830        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19831        assert_eq!(results[0].1.turns, 0);
19832        assert!(FailClass::of(&results[0].2).is_none());
19833        assert_eq!(
19834            st.resource_block.as_ref().unwrap().kind,
19835            "repository coordination"
19836        );
19837        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19838        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19839        drop(external);
19840        let editor = crate::repo_lock::SeatLease::for_checkout(
19841            tmp.path(),
19842            true,
19843            crate::config::test_coordination().as_ref(),
19844        )
19845        .await
19846        .unwrap();
19847        let pause = Pause::new();
19848        let interrupt = Pause::new();
19849        let prompts = Prompts::default();
19850        let ctx = WaveCtx {
19851            run: "r",
19852            node: "review",
19853            prompts: &prompts,
19854            cache: None,
19855            round: None,
19856            carry_seats: false,
19857            pauses: Some((&pause, &interrupt)),
19858        };
19859        let (results, ()) = tokio::join!(
19860            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19861            async {
19862                tokio::time::sleep(Duration::from_millis(50)).await;
19863                pause.park();
19864            }
19865        );
19866        assert!(
19867            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19868        );
19869        assert_eq!(results[0].1.turns, 0);
19870        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19871        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19872        drop(editor);
19873    }
19874
19875    #[tokio::test]
19876    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19877        crate::run::pin_test_home();
19878        let tmp = tempfile::tempdir().unwrap();
19879        let cache = tmp.path().join("target-b");
19880        let _held = hold(&cache);
19881        let mut st = state(1);
19882        let results = run(
19883            vec![job(
19884                sh("alpha", "echo ok"),
19885                tmp.path(),
19886                Duration::from_secs(1),
19887            )],
19888            Arc::new(Semaphore::new(1)),
19889            Some(&cache),
19890            &mut st,
19891        )
19892        .await;
19893        let (_, seat, out) = &results[0];
19894        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19895        assert!(wave_blocked(&results));
19896        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19897        assert_eq!(seat.turns, 0, "nothing was asked");
19898        assert!(
19899            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19900            "no agent was spawned"
19901        );
19902        assert!(FailClass::of(out).is_none(), "not a failure class");
19903        let block = st.resource_block.as_ref().expect("recorded for the resume");
19904        assert_eq!(block.kind, "shared build cache");
19905        assert!(matches!(
19906            block.cause,
19907            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
19908        ));
19909        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19910        assert!(st.active.is_empty());
19911    }
19912
19913    #[tokio::test]
19914    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19915        crate::run::pin_test_home();
19916        let tmp = tempfile::tempdir().unwrap();
19917        let mut st = state(1);
19918        let results = run(
19919            vec![job(
19920                sh("alpha", "echo ok"),
19921                tmp.path(),
19922                Duration::from_secs(1),
19923            )],
19924            Arc::new(Semaphore::new(0)),
19925            None,
19926            &mut st,
19927        )
19928        .await;
19929        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19930        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19931        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19932    }
19933
19934    #[tokio::test]
19935    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19936        crate::run::pin_test_home();
19937        let tmp = tempfile::tempdir().unwrap();
19938        let mut st = state(60);
19939        let results = run(
19940            vec![job(
19941                sh("alpha", "sleep 30"),
19942                tmp.path(),
19943                Duration::from_secs(1),
19944            )],
19945            Arc::new(Semaphore::new(1)),
19946            None,
19947            &mut st,
19948        )
19949        .await;
19950        let out = &results[0].2;
19951        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19952        let class = FailClass::of(out).expect("a failure");
19953        assert_eq!(class, FailClass::Timeout);
19954        assert!(should_hand_over(None, &class), "existing handover policy");
19955        assert!(st.resource_block.is_none());
19956    }
19957}